From d24b4a56974b8bbe032e95f1e2ee6bcaf3316b3c Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 19 Aug 2025 11:23:51 -0500 Subject: [PATCH 01/53] PYTHON-5503 Use uv to install just in GitHub Actions (#2490) --- .github/workflows/test-python.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test-python.yml b/.github/workflows/test-python.yml index 96729e3a6e..11255f9e49 100644 --- a/.github/workflows/test-python.yml +++ b/.github/workflows/test-python.yml @@ -22,13 +22,13 @@ jobs: - uses: actions/checkout@v4 with: persist-credentials: false - - name: Install just - uses: extractions/setup-just@e33e0265a09d6d736e2ee1e0eb685ef1de4669ff # v3 - name: Install uv uses: astral-sh/setup-uv@e92bafb6253dcd438e0484186d7669ea7a8ca1cc # v5 with: enable-cache: true python-version: "3.9" + - name: Install just + run: uv tool install rust-just - name: Install Python dependencies run: | just install @@ -83,13 +83,13 @@ jobs: - uses: actions/checkout@v4 with: persist-credentials: false - - name: Install just - uses: extractions/setup-just@e33e0265a09d6d736e2ee1e0eb685ef1de4669ff # v3 - name: Install uv uses: astral-sh/setup-uv@e92bafb6253dcd438e0484186d7669ea7a8ca1cc # v5 with: enable-cache: true python-version: "3.9" + - name: Install just + run: uv tool install rust-just - id: setup-mongodb uses: mongodb-labs/drivers-evergreen-tools@master with: @@ -114,7 +114,7 @@ jobs: enable-cache: true python-version: "3.9" - name: Install just - uses: extractions/setup-just@e33e0265a09d6d736e2ee1e0eb685ef1de4669ff # v3 + run: uv tool install rust-just - name: Install dependencies run: just install - name: Build docs @@ -133,7 +133,7 @@ jobs: enable-cache: true python-version: "3.9" - name: Install just - uses: extractions/setup-just@e33e0265a09d6d736e2ee1e0eb685ef1de4669ff # v3 + run: uv tool install rust-just - name: Install dependencies run: just install - name: Build docs @@ -155,7 +155,7 @@ jobs: enable-cache: true python-version: "${{matrix.python}}" - name: Install just - uses: extractions/setup-just@e33e0265a09d6d736e2ee1e0eb685ef1de4669ff # v3 + run: uv tool install rust-just - name: Install dependencies run: | just install From 3a26119eb332fed1cded500a137eb54a129660ad Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 19 Aug 2025 11:26:11 -0500 Subject: [PATCH 02/53] PYTHON-5502 Fix c extensions on OIDC VMs (#2489) --- .evergreen/run-mongodb-oidc-test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.evergreen/run-mongodb-oidc-test.sh b/.evergreen/run-mongodb-oidc-test.sh index 1a1cd81a8b..b34013a6ac 100755 --- a/.evergreen/run-mongodb-oidc-test.sh +++ b/.evergreen/run-mongodb-oidc-test.sh @@ -8,7 +8,7 @@ if [ ${OIDC_ENV} == "k8s" ]; then SUB_TEST_NAME=$K8S_VARIANT-remote else SUB_TEST_NAME=$OIDC_ENV-remote - apt-get install -y python3-dev build-essential + sudo apt-get install -y python3-dev build-essential fi bash ./.evergreen/just.sh setup-tests auth_oidc $SUB_TEST_NAME From db3d3c702225431f74d607e9eb11fbeb528c27cb Mon Sep 17 00:00:00 2001 From: Iris <58442094+sleepyStick@users.noreply.github.com> Date: Tue, 19 Aug 2025 17:46:25 -0700 Subject: [PATCH 03/53] Prep for 4.14.1 release (#2495) [master] (#2496) --- doc/changelog.rst | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/doc/changelog.rst b/doc/changelog.rst index a553be0144..e41ecc7e1b 100644 --- a/doc/changelog.rst +++ b/doc/changelog.rst @@ -1,5 +1,20 @@ Changelog ========= +Changes in Version 4.14.1 (2025/08/19) +-------------------------------------- + +Version 4.14.1 is a bug fix release. + + - Fixed a bug in ``MongoClient.append_metadata()`` and ``AsyncMongoClient.append_metadata()`` + that allowed duplicate ``DriverInfo.name`` to be appended to the metadata. + +Issues Resolved +............... + +See the `PyMongo 4.14.1 release notes in JIRA`_ for the list of resolved issues +in this release. + +.. _PyMongo 4.14.1 release notes in JIRA: https://jira.mongodb.org/secure/ReleaseNote.jspa?projectId=10004&version=45256 Changes in Version 4.14.0 (2025/08/06) -------------------------------------- @@ -34,6 +49,14 @@ PyMongo 4.14 brings a number of changes including: - Changed :meth:`~pymongo.uri_parser.parse_uri`'s ``options`` return value to be type ``dict`` instead of ``_CaseInsensitiveDictionary``. +Issues Resolved +............... + +See the `PyMongo 4.14 release notes in JIRA`_ for the list of resolved issues +in this release. + +.. _PyMongo 4.14 release notes in JIRA: https://jira.mongodb.org/secure/ReleaseNote.jspa?projectId=10004&version=43041 + Changes in Version 4.13.2 (2025/06/17) -------------------------------------- From f7b94be0dbd263c8e2e570c2ae425946a075a08a Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 20 Aug 2025 08:58:20 -0500 Subject: [PATCH 04/53] PYTHON-5143 Support auto encryption in unified tests (#2488) --- .evergreen/remove-unimplemented-tests.sh | 5 - test/__init__.py | 5 +- test/asynchronous/__init__.py | 5 +- test/asynchronous/helpers.py | 248 +------------ test/asynchronous/test_encryption.py | 46 +-- test/asynchronous/unified_format.py | 40 +- .../unified/fle2v2-BypassQueryAnalysis.json | 322 ++++++++++++++++ ...EncryptedFields-vs-EncryptedFieldsMap.json | 256 +++++++++++++ .../spec/unified/localSchema.json | 343 ++++++++++++++++++ .../spec/unified/maxWireVersion.json | 101 ++++++ test/helpers.py | 248 +------------ test/helpers_shared.py | 271 ++++++++++++++ test/test_encryption.py | 46 +-- test/test_uri_spec.py | 2 +- .../valid-pass/poc-queryable-encryption.json | 193 ++++++++++ test/unified_format.py | 40 +- test/unified_format_shared.py | 15 +- 17 files changed, 1627 insertions(+), 559 deletions(-) create mode 100644 test/client-side-encryption/spec/unified/fle2v2-BypassQueryAnalysis.json create mode 100644 test/client-side-encryption/spec/unified/fle2v2-EncryptedFields-vs-EncryptedFieldsMap.json create mode 100644 test/client-side-encryption/spec/unified/localSchema.json create mode 100644 test/client-side-encryption/spec/unified/maxWireVersion.json create mode 100644 test/helpers_shared.py create mode 100644 test/unified-test-format/valid-pass/poc-queryable-encryption.json diff --git a/.evergreen/remove-unimplemented-tests.sh b/.evergreen/remove-unimplemented-tests.sh index 92685ab2b7..fd50010138 100755 --- a/.evergreen/remove-unimplemented-tests.sh +++ b/.evergreen/remove-unimplemented-tests.sh @@ -3,11 +3,6 @@ PYMONGO=$(dirname "$(cd "$(dirname "$0")" || exit; pwd)") rm $PYMONGO/test/transactions/legacy/errors-client.json # PYTHON-1894 rm $PYMONGO/test/connection_monitoring/wait-queue-fairness.json # PYTHON-1873 -rm $PYMONGO/test/client-side-encryption/spec/unified/fle2v2-BypassQueryAnalysis.json # PYTHON-5143 -rm $PYMONGO/test/client-side-encryption/spec/unified/fle2v2-EncryptedFields-vs-EncryptedFieldsMap.json # PYTHON-5143 -rm $PYMONGO/test/client-side-encryption/spec/unified/localSchema.json # PYTHON-5143 -rm $PYMONGO/test/client-side-encryption/spec/unified/maxWireVersion.json # PYTHON-5143 -rm $PYMONGO/test/unified-test-format/valid-pass/poc-queryable-encryption.json # PYTHON-5143 rm $PYMONGO/test/discovery_and_monitoring/unified/pool-clear-application-error.json # PYTHON-4918 rm $PYMONGO/test/discovery_and_monitoring/unified/pool-clear-checkout-error.json # PYTHON-4918 rm $PYMONGO/test/discovery_and_monitoring/unified/pool-clear-min-pool-size-error.json # PYTHON-4918 diff --git a/test/__init__.py b/test/__init__.py index 95c2d7ee9d..12660e3a4a 100644 --- a/test/__init__.py +++ b/test/__init__.py @@ -59,7 +59,8 @@ sys.path[0:0] = [""] -from test.helpers import ( +from test.helpers import client_knobs, global_knobs +from test.helpers_shared import ( COMPRESSORS, IS_SRV, MONGODB_API_VERSION, @@ -67,10 +68,8 @@ TEST_LOADBALANCER, TLS_OPTIONS, SystemCertsPatcher, - client_knobs, db_pwd, db_user, - global_knobs, host, is_server_resolvable, port, diff --git a/test/asynchronous/__init__.py b/test/asynchronous/__init__.py index 96769dc9c5..7b594b184d 100644 --- a/test/asynchronous/__init__.py +++ b/test/asynchronous/__init__.py @@ -59,7 +59,8 @@ sys.path[0:0] = [""] -from test.helpers import ( +from test.asynchronous.helpers import client_knobs, global_knobs +from test.helpers_shared import ( COMPRESSORS, IS_SRV, MONGODB_API_VERSION, @@ -67,10 +68,8 @@ TEST_LOADBALANCER, TLS_OPTIONS, SystemCertsPatcher, - client_knobs, db_pwd, db_user, - global_knobs, host, is_server_resolvable, port, diff --git a/test/asynchronous/helpers.py b/test/asynchronous/helpers.py index bcb004af51..892c629631 100644 --- a/test/asynchronous/helpers.py +++ b/test/asynchronous/helpers.py @@ -12,137 +12,22 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Shared constants and helper methods for pymongo, bson, and gridfs test suites.""" +"""Shared helper methods for pymongo, bson, and gridfs test suites.""" from __future__ import annotations import asyncio -import base64 -import gc -import multiprocessing -import os -import signal -import socket -import subprocess -import sys import threading -import time import traceback -import unittest -import warnings -from inspect import iscoroutinefunction - -from pymongo._asyncio_task import create_task - -try: - import ipaddress - - HAVE_IPADDRESS = True -except ImportError: - HAVE_IPADDRESS = False from functools import wraps -from typing import Any, Callable, Dict, Generator, Optional, no_type_check -from unittest import SkipTest +from typing import Optional, no_type_check -from bson.son import SON -from pymongo import common, message +from bson import SON +from pymongo import common +from pymongo._asyncio_task import create_task from pymongo.read_preferences import ReadPreference -from pymongo.ssl_support import HAVE_SSL, _ssl # type:ignore[attr-defined] -from pymongo.synchronous.uri_parser import parse_uri - -if HAVE_SSL: - import ssl _IS_SYNC = False -# Enable debug output for uncollectable objects. PyPy does not have set_debug. -if hasattr(gc, "set_debug"): - gc.set_debug( - gc.DEBUG_UNCOLLECTABLE | getattr(gc, "DEBUG_OBJECTS", 0) | getattr(gc, "DEBUG_INSTANCES", 0) - ) - -# The host and port of a single mongod or mongos, or the seed host -# for a replica set. -host = os.environ.get("DB_IP", "localhost") -port = int(os.environ.get("DB_PORT", 27017)) -IS_SRV = "mongodb+srv" in host - -db_user = os.environ.get("DB_USER", "user") -db_pwd = os.environ.get("DB_PASSWORD", "password") - -CERT_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), "certificates") -CLIENT_PEM = os.environ.get("CLIENT_PEM", os.path.join(CERT_PATH, "client.pem")) -CA_PEM = os.environ.get("CA_PEM", os.path.join(CERT_PATH, "ca.pem")) - -TLS_OPTIONS: Dict = {"tls": True} -if CLIENT_PEM: - TLS_OPTIONS["tlsCertificateKeyFile"] = CLIENT_PEM -if CA_PEM: - TLS_OPTIONS["tlsCAFile"] = CA_PEM - -COMPRESSORS = os.environ.get("COMPRESSORS") -MONGODB_API_VERSION = os.environ.get("MONGODB_API_VERSION") -TEST_LOADBALANCER = bool(os.environ.get("TEST_LOAD_BALANCER")) -SINGLE_MONGOS_LB_URI = os.environ.get("SINGLE_MONGOS_LB_URI") -MULTI_MONGOS_LB_URI = os.environ.get("MULTI_MONGOS_LB_URI") - -if TEST_LOADBALANCER: - res = parse_uri(SINGLE_MONGOS_LB_URI or "") - host, port = res["nodelist"][0] - db_user = res["username"] or db_user - db_pwd = res["password"] or db_pwd - - -# Shared KMS data. -LOCAL_MASTER_KEY = base64.b64decode( - b"Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ" - b"5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk" -) -AWS_CREDS = { - "accessKeyId": os.environ.get("FLE_AWS_KEY", ""), - "secretAccessKey": os.environ.get("FLE_AWS_SECRET", ""), -} -AWS_CREDS_2 = { - "accessKeyId": os.environ.get("FLE_AWS_KEY2", ""), - "secretAccessKey": os.environ.get("FLE_AWS_SECRET2", ""), -} -AZURE_CREDS = { - "tenantId": os.environ.get("FLE_AZURE_TENANTID", ""), - "clientId": os.environ.get("FLE_AZURE_CLIENTID", ""), - "clientSecret": os.environ.get("FLE_AZURE_CLIENTSECRET", ""), -} -GCP_CREDS = { - "email": os.environ.get("FLE_GCP_EMAIL", ""), - "privateKey": os.environ.get("FLE_GCP_PRIVATEKEY", ""), -} -KMIP_CREDS = {"endpoint": os.environ.get("FLE_KMIP_ENDPOINT", "localhost:5698")} - -# Ensure Evergreen metadata doesn't result in truncation -os.environ.setdefault("MONGOB_LOG_MAX_DOCUMENT_LENGTH", "2000") - - -def is_server_resolvable(): - """Returns True if 'server' is resolvable.""" - socket_timeout = socket.getdefaulttimeout() - socket.setdefaulttimeout(1) - try: - try: - socket.gethostbyname("server") - return True - except OSError: - return False - finally: - socket.setdefaulttimeout(socket_timeout) - - -def _create_user(authdb, user, pwd=None, roles=None, **kwargs): - cmd = SON([("createUser", user)]) - # X509 doesn't use a password - if pwd: - cmd["pwd"] = pwd - cmd["roles"] = roles or ["root"] - cmd.update(**kwargs) - return authdb.command(cmd) - async def async_repl_set_step_down(client, **kwargs): """Run replSetStepDown, first unfreezing a secondary with replSetFreeze.""" @@ -237,133 +122,10 @@ def __del__(self): raise Exception(msg) -def _all_users(db): - return {u["user"] for u in db.command("usersInfo").get("users", [])} - - -def sanitize_cmd(cmd): - cp = cmd.copy() - cp.pop("$clusterTime", None) - cp.pop("$db", None) - cp.pop("$readPreference", None) - cp.pop("lsid", None) - if MONGODB_API_VERSION: - # Stable API parameters - cp.pop("apiVersion", None) - # OP_MSG encoding may move the payload type one field to the - # end of the command. Do the same here. - name = next(iter(cp)) - try: - identifier = message._FIELD_MAP[name] - docs = cp.pop(identifier) - cp[identifier] = docs - except KeyError: - pass - return cp - - -def sanitize_reply(reply): - cp = reply.copy() - cp.pop("$clusterTime", None) - cp.pop("operationTime", None) - return cp - - -def print_thread_tracebacks() -> None: - """Print all Python thread tracebacks.""" - for thread_id, frame in sys._current_frames().items(): - sys.stderr.write(f"\n--- Traceback for thread {thread_id} ---\n") - traceback.print_stack(frame, file=sys.stderr) - - -def print_thread_stacks(pid: int) -> None: - """Print all C-level thread stacks for a given process id.""" - if sys.platform == "darwin": - cmd = ["lldb", "--attach-pid", f"{pid}", "--batch", "--one-line", '"thread backtrace all"'] - else: - cmd = ["gdb", f"--pid={pid}", "--batch", '--eval-command="thread apply all bt"'] - - try: - res = subprocess.run( - cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, encoding="utf-8" - ) - except Exception as exc: - sys.stderr.write(f"Could not print C-level thread stacks because {cmd[0]} failed: {exc}") - else: - sys.stderr.write(res.stdout) - - # Global knobs to speed up the test suite. global_knobs = client_knobs(events_queue_frequency=0.05) -def _get_executors(topology): - executors = [] - for server in topology._servers.values(): - # Some MockMonitor do not have an _executor. - if hasattr(server._monitor, "_executor"): - executors.append(server._monitor._executor) - if hasattr(server._monitor, "_rtt_monitor"): - executors.append(server._monitor._rtt_monitor._executor) - executors.append(topology._Topology__events_executor) - if topology._srv_monitor: - executors.append(topology._srv_monitor._executor) - - return [e for e in executors if e is not None] - - -def print_running_topology(topology): - running = [e for e in _get_executors(topology) if not e._stopped] - if running: - print( - "WARNING: found Topology with running threads:\n" - f" Threads: {running}\n" - f" Topology: {topology}\n" - f" Creation traceback:\n{topology._settings._stack}" - ) - - -def test_cases(suite): - """Iterator over all TestCases within a TestSuite.""" - for suite_or_case in suite._tests: - if isinstance(suite_or_case, unittest.TestCase): - # unittest.TestCase - yield suite_or_case - else: - # unittest.TestSuite - yield from test_cases(suite_or_case) - - -# Helper method to workaround https://bugs.python.org/issue21724 -def clear_warning_registry(): - """Clear the __warningregistry__ for all modules.""" - for _, module in list(sys.modules.items()): - if hasattr(module, "__warningregistry__"): - module.__warningregistry__ = {} # type:ignore[attr-defined] - - -class SystemCertsPatcher: - def __init__(self, ca_certs): - if ( - ssl.OPENSSL_VERSION.lower().startswith("libressl") - and sys.platform == "darwin" - and not _ssl.IS_PYOPENSSL - ): - raise SkipTest( - "LibreSSL on OSX doesn't support setting CA certificates " - "using SSL_CERT_FILE environment variable." - ) - self.original_certs = os.environ.get("SSL_CERT_FILE") - # Tell OpenSSL where CA certificates live. - os.environ["SSL_CERT_FILE"] = ca_certs - - def disable(self): - if self.original_certs is None: - os.environ.pop("SSL_CERT_FILE") - else: - os.environ["SSL_CERT_FILE"] = self.original_certs - - if _IS_SYNC: PARENT = threading.Thread else: diff --git a/test/asynchronous/test_encryption.py b/test/asynchronous/test_encryption.py index f6afa4b2a3..337dba0f64 100644 --- a/test/asynchronous/test_encryption.py +++ b/test/asynchronous/test_encryption.py @@ -57,11 +57,14 @@ from test.asynchronous.test_bulk import AsyncBulkTestBase from test.asynchronous.unified_format import generate_test_classes from test.asynchronous.utils_spec_runner import AsyncSpecRunner -from test.helpers import ( +from test.helpers_shared import ( + ALL_KMS_PROVIDERS, AWS_CREDS, + AWS_TEMP_CREDS, AZURE_CREDS, CA_PEM, CLIENT_PEM, + DEFAULT_KMS_TLS, GCP_CREDS, KMIP_CREDS, LOCAL_MASTER_KEY, @@ -204,7 +207,7 @@ async def test_init_kms_tls_options(self): opts = AutoEncryptionOpts( {}, "k.d", - kms_tls_options={"kmip": {"tlsCAFile": CA_PEM, "tlsCertificateKeyFile": CLIENT_PEM}}, + kms_tls_options=DEFAULT_KMS_TLS, ) _kms_ssl_contexts = _parse_kms_tls_options(opts._kms_tls_options, _IS_SYNC) ctx = _kms_ssl_contexts["kmip"] @@ -616,17 +619,10 @@ async def test_with_statement(self): # Spec tests -AWS_TEMP_CREDS = { - "accessKeyId": os.environ.get("CSFLE_AWS_TEMP_ACCESS_KEY_ID", ""), - "secretAccessKey": os.environ.get("CSFLE_AWS_TEMP_SECRET_ACCESS_KEY", ""), - "sessionToken": os.environ.get("CSFLE_AWS_TEMP_SESSION_TOKEN", ""), -} - AWS_TEMP_NO_SESSION_CREDS = { "accessKeyId": os.environ.get("CSFLE_AWS_TEMP_ACCESS_KEY_ID", ""), "secretAccessKey": os.environ.get("CSFLE_AWS_TEMP_SECRET_ACCESS_KEY", ""), } -KMS_TLS_OPTS = {"kmip": {"tlsCAFile": CA_PEM, "tlsCertificateKeyFile": CLIENT_PEM}} class AsyncTestSpec(AsyncSpecRunner): @@ -663,7 +659,7 @@ def parse_auto_encrypt_opts(self, opts): self.skipTest("GCP environment credentials are not set") if "kmip" in kms_providers: kms_providers["kmip"] = KMIP_CREDS - opts["kms_tls_options"] = KMS_TLS_OPTS + opts["kms_tls_options"] = DEFAULT_KMS_TLS if "key_vault_namespace" not in opts: opts["key_vault_namespace"] = "keyvault.datakeys" if "extra_options" in opts: @@ -757,14 +753,6 @@ async def run_scenario(self): ) # Prose Tests -ALL_KMS_PROVIDERS = { - "aws": AWS_CREDS, - "azure": AZURE_CREDS, - "gcp": GCP_CREDS, - "kmip": KMIP_CREDS, - "local": {"key": LOCAL_MASTER_KEY}, -} - LOCAL_KEY_ID = Binary(base64.b64decode(b"LOCALAAAAAAAAAAAAAAAAA=="), UUID_SUBTYPE) AWS_KEY_ID = Binary(base64.b64decode(b"AWSAAAAAAAAAAAAAAAAAAA=="), UUID_SUBTYPE) AZURE_KEY_ID = Binary(base64.b64decode(b"AZUREAAAAAAAAAAAAAAAAA=="), UUID_SUBTYPE) @@ -851,13 +839,17 @@ async def asyncSetUp(self): self.KMS_PROVIDERS, "keyvault.datakeys", schema_map=schemas, - kms_tls_options=KMS_TLS_OPTS, + kms_tls_options=DEFAULT_KMS_TLS, ) self.client_encrypted = await self.async_rs_or_single_client( auto_encryption_opts=opts, uuidRepresentation="standard" ) self.client_encryption = self.create_client_encryption( - self.KMS_PROVIDERS, "keyvault.datakeys", self.client, OPTS, kms_tls_options=KMS_TLS_OPTS + self.KMS_PROVIDERS, + "keyvault.datakeys", + self.client, + OPTS, + kms_tls_options=DEFAULT_KMS_TLS, ) self.listener.reset() @@ -1066,7 +1058,7 @@ async def _test_corpus(self, opts): "keyvault.datakeys", async_client_context.client, OPTS, - kms_tls_options=KMS_TLS_OPTS, + kms_tls_options=DEFAULT_KMS_TLS, ) corpus = self.fix_up_curpus(json_data("corpus", "corpus.json")) @@ -1158,7 +1150,7 @@ async def _test_corpus(self, opts): async def test_corpus(self): opts = AutoEncryptionOpts( - self.kms_providers(), "keyvault.datakeys", kms_tls_options=KMS_TLS_OPTS + self.kms_providers(), "keyvault.datakeys", kms_tls_options=DEFAULT_KMS_TLS ) await self._test_corpus(opts) @@ -1169,7 +1161,7 @@ async def test_corpus_local_schema(self): self.kms_providers(), "keyvault.datakeys", schema_map=schemas, - kms_tls_options=KMS_TLS_OPTS, + kms_tls_options=DEFAULT_KMS_TLS, ) await self._test_corpus(opts) @@ -1300,7 +1292,7 @@ async def asyncSetUp(self): key_vault_namespace="keyvault.datakeys", key_vault_client=async_client_context.client, codec_options=OPTS, - kms_tls_options=KMS_TLS_OPTS, + kms_tls_options=DEFAULT_KMS_TLS, ) kms_providers_invalid = copy.deepcopy(kms_providers) @@ -1312,7 +1304,7 @@ async def asyncSetUp(self): key_vault_namespace="keyvault.datakeys", key_vault_client=async_client_context.client, codec_options=OPTS, - kms_tls_options=KMS_TLS_OPTS, + kms_tls_options=DEFAULT_KMS_TLS, ) self._kmip_host_error = None self._invalid_host_error = None @@ -2752,7 +2744,7 @@ async def run_test(self, src_provider, dst_provider): key_vault_client=self.client, key_vault_namespace="keyvault.datakeys", kms_providers=ALL_KMS_PROVIDERS, - kms_tls_options=KMS_TLS_OPTS, + kms_tls_options=DEFAULT_KMS_TLS, codec_options=OPTS, ) @@ -2772,7 +2764,7 @@ async def run_test(self, src_provider, dst_provider): key_vault_client=client2, key_vault_namespace="keyvault.datakeys", kms_providers=ALL_KMS_PROVIDERS, - kms_tls_options=KMS_TLS_OPTS, + kms_tls_options=DEFAULT_KMS_TLS, codec_options=OPTS, ) diff --git a/test/asynchronous/unified_format.py b/test/asynchronous/unified_format.py index 09bf7e83ea..b06654b328 100644 --- a/test/asynchronous/unified_format.py +++ b/test/asynchronous/unified_format.py @@ -37,6 +37,7 @@ ) from test.asynchronous.utils import async_get_pool, flaky from test.asynchronous.utils_spec_runner import SpecRunnerTask +from test.helpers_shared import ALL_KMS_PROVIDERS, DEFAULT_KMS_TLS from test.unified_format_shared import ( KMS_TLS_OPTS, PLACEHOLDER_MAP, @@ -61,6 +62,8 @@ from test.version import Version from typing import Any, Dict, List, Mapping, Optional +import pytest + import pymongo from bson import SON, json_util from bson.codec_options import DEFAULT_CODEC_OPTIONS @@ -76,7 +79,7 @@ from pymongo.asynchronous.encryption import AsyncClientEncryption from pymongo.asynchronous.helpers import anext from pymongo.driver_info import DriverInfo -from pymongo.encryption_options import _HAVE_PYMONGOCRYPT +from pymongo.encryption_options import _HAVE_PYMONGOCRYPT, AutoEncryptionOpts from pymongo.errors import ( AutoReconnect, BulkWriteError, @@ -259,6 +262,23 @@ async def _create_entity(self, entity_spec, uri=None): kwargs: dict = {} observe_events = spec.get("observeEvents", []) + if "autoEncryptOpts" in spec: + auto_encrypt_opts = spec["autoEncryptOpts"].copy() + auto_encrypt_kwargs: dict = dict(kms_tls_options=DEFAULT_KMS_TLS) + kms_providers = ALL_KMS_PROVIDERS.copy() + key_vault_namespace = auto_encrypt_opts.pop("keyVaultNamespace") + for provider_name, provider_value in auto_encrypt_opts.pop("kmsProviders").items(): + kms_providers[provider_name].update(provider_value) + extra_opts = auto_encrypt_opts.pop("extraOptions", {}) + for key, value in extra_opts.items(): + auto_encrypt_kwargs[camel_to_snake(key)] = value + for key, value in auto_encrypt_opts.items(): + auto_encrypt_kwargs[camel_to_snake(key)] = value + auto_encryption_opts = AutoEncryptionOpts( + kms_providers, key_vault_namespace, **auto_encrypt_kwargs + ) + kwargs["auto_encryption_opts"] = auto_encryption_opts + # The unified tests use topologyOpeningEvent, we use topologyOpenedEvent for i in range(len(observe_events)): if "topologyOpeningEvent" == observe_events[i]: @@ -430,7 +450,7 @@ class UnifiedSpecTestMixinV1(AsyncIntegrationTest): a class attribute ``TEST_SPEC``. """ - SCHEMA_VERSION = Version.from_string("1.22") + SCHEMA_VERSION = Version.from_string("1.23") RUN_ON_LOAD_BALANCER = True TEST_SPEC: Any TEST_PATH = "" # This gets filled in by generate_test_classes @@ -462,6 +482,13 @@ async def insert_initial_data(self, initial_data): wc = WriteConcern(w="majority") else: wc = WriteConcern(w=1) + + # Remove any encryption collections associated with the collection. + collections = await db.list_collection_names() + for collection in collections: + if collection in [f"enxcol_.{coll_name}.esc", f"enxcol_.{coll_name}.ecoc"]: + await db.drop_collection(collection) + if documents: if opts: await db.create_collection(coll_name, **opts) @@ -1516,7 +1543,14 @@ class SpecTestBase(with_metaclass(UnifiedSpecTestMeta)): # type: ignore TEST_SPEC = test_spec EXPECTED_FAILURES = expected_failures - return SpecTestBase + base = SpecTestBase + + # Add "encryption" marker if the "csfle" runOnRequirement is set. + for req in test_spec.get("runOnRequirements", []): + if req.get("csfle", False): + base = pytest.mark.encryption(base) + + return base for dirpath, _, filenames in os.walk(test_path): dirname = os.path.split(dirpath)[-1] diff --git a/test/client-side-encryption/spec/unified/fle2v2-BypassQueryAnalysis.json b/test/client-side-encryption/spec/unified/fle2v2-BypassQueryAnalysis.json new file mode 100644 index 0000000000..0817508f8f --- /dev/null +++ b/test/client-side-encryption/spec/unified/fle2v2-BypassQueryAnalysis.json @@ -0,0 +1,322 @@ +{ + "description": "fle2v2-BypassQueryAnalysis", + "schemaVersion": "1.23", + "runOnRequirements": [ + { + "minServerVersion": "7.0.0", + "serverless": "forbid", + "csfle": true, + "topologies": [ + "replicaset", + "sharded", + "load-balanced" + ] + } + ], + "createEntities": [ + { + "client": { + "id": "client0", + "autoEncryptOpts": { + "kmsProviders": { + "local": { + "key": "Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk" + } + }, + "keyVaultNamespace": "keyvault.datakeys", + "bypassQueryAnalysis": true + }, + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "database": { + "id": "encryptedDB", + "client": "client0", + "databaseName": "default" + } + }, + { + "collection": { + "id": "encryptedColl", + "database": "encryptedDB", + "collectionName": "default" + } + }, + { + "client": { + "id": "client1" + } + }, + { + "database": { + "id": "unencryptedDB", + "client": "client1", + "databaseName": "default" + } + }, + { + "collection": { + "id": "unencryptedColl", + "database": "unencryptedDB", + "collectionName": "default" + } + } + ], + "initialData": [ + { + "databaseName": "keyvault", + "collectionName": "datakeys", + "documents": [ + { + "_id": { + "$binary": { + "base64": "EjRWeBI0mHYSNBI0VniQEg==", + "subType": "04" + } + }, + "keyMaterial": { + "$binary": { + "base64": "sHe0kz57YW7v8g9VP9sf/+K1ex4JqKc5rf/URX3n3p8XdZ6+15uXPaSayC6adWbNxkFskuMCOifDoTT+rkqMtFkDclOy884RuGGtUysq3X7zkAWYTKi8QAfKkajvVbZl2y23UqgVasdQu3OVBQCrH/xY00nNAs/52e958nVjBuzQkSb1T8pKJAyjZsHJ60+FtnfafDZSTAIBJYn7UWBCwQ==", + "subType": "00" + } + }, + "creationDate": { + "$date": { + "$numberLong": "1648914851981" + } + }, + "updateDate": { + "$date": { + "$numberLong": "1648914851981" + } + }, + "status": { + "$numberInt": "0" + }, + "masterKey": { + "provider": "local" + } + } + ] + }, + { + "databaseName": "default", + "collectionName": "default", + "documents": [], + "createOptions": { + "encryptedFields": { + "fields": [ + { + "keyId": { + "$binary": { + "base64": "EjRWeBI0mHYSNBI0VniQEg==", + "subType": "04" + } + }, + "path": "encryptedIndexed", + "bsonType": "string", + "queries": { + "queryType": "equality", + "contention": { + "$numberLong": "0" + } + } + }, + { + "keyId": { + "$binary": { + "base64": "q83vqxI0mHYSNBI0VniQEg==", + "subType": "04" + } + }, + "path": "encryptedUnindexed", + "bsonType": "string" + } + ] + } + } + } + ], + "tests": [ + { + "description": "BypassQueryAnalysis decrypts", + "operations": [ + { + "object": "encryptedColl", + "name": "insertOne", + "arguments": { + "document": { + "_id": 1, + "encryptedIndexed": { + "$binary": { + "base64": "C18BAAAFZAAgAAAAANnt+eLTkv4GdDPl8IAfJOvTzArOgFJQ2S/DcLza4W0DBXMAIAAAAAD2u+omZme3P2gBPehMQyQHQ153tPN1+z7bksYA9jKTpAVwADAAAAAAUnCOQqIvmR65YKyYnsiVfVrg9hwUVO3RhhKExo3RWOzgaS0QdsBL5xKFS0JhZSoWBXUAEAAAAAQSNFZ4EjSYdhI0EjRWeJASEHQAAgAAAAV2AFAAAAAAEjRWeBI0mHYSNBI0VniQEpQbp/ZJpWBKeDtKLiXb0P2E9wvc0g3f373jnYQYlJquOrlPOoEy3ngsHPJuSUijvWDsrQzqYa349K7G/66qaXEFZQAgAAAAAOuac/eRLYakKX6B0vZ1r3QodOQFfjqJD+xlGiPu4/PsBWwAIAAAAACkm0o9bj6j0HuADKc0svbqO2UHj6GrlNdF6yKNxh63xRJrAAAAAAAAAAAAAA==", + "subType": "06" + } + } + } + } + }, + { + "object": "encryptedColl", + "name": "find", + "arguments": { + "filter": { + "_id": 1 + } + }, + "expectResult": [ + { + "_id": 1, + "encryptedIndexed": "123" + } + ] + }, + { + "object": "unencryptedColl", + "name": "find", + "arguments": { + "filter": {} + }, + "expectResult": [ + { + "_id": 1, + "encryptedIndexed": { + "$$type": "binData" + }, + "__safeContent__": [ + { + "$binary": { + "base64": "31eCYlbQoVboc5zwC8IoyJVSkag9PxREka8dkmbXJeY=", + "subType": "00" + } + } + ] + } + ] + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "listCollections": 1, + "filter": { + "name": "default" + } + }, + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "command": { + "insert": "default", + "documents": [ + { + "_id": 1, + "encryptedIndexed": { + "$binary": { + "base64": "C18BAAAFZAAgAAAAANnt+eLTkv4GdDPl8IAfJOvTzArOgFJQ2S/DcLza4W0DBXMAIAAAAAD2u+omZme3P2gBPehMQyQHQ153tPN1+z7bksYA9jKTpAVwADAAAAAAUnCOQqIvmR65YKyYnsiVfVrg9hwUVO3RhhKExo3RWOzgaS0QdsBL5xKFS0JhZSoWBXUAEAAAAAQSNFZ4EjSYdhI0EjRWeJASEHQAAgAAAAV2AFAAAAAAEjRWeBI0mHYSNBI0VniQEpQbp/ZJpWBKeDtKLiXb0P2E9wvc0g3f373jnYQYlJquOrlPOoEy3ngsHPJuSUijvWDsrQzqYa349K7G/66qaXEFZQAgAAAAAOuac/eRLYakKX6B0vZ1r3QodOQFfjqJD+xlGiPu4/PsBWwAIAAAAACkm0o9bj6j0HuADKc0svbqO2UHj6GrlNdF6yKNxh63xRJrAAAAAAAAAAAAAA==", + "subType": "06" + } + } + } + ], + "ordered": true, + "encryptionInformation": { + "type": 1, + "schema": { + "default.default": { + "escCollection": "enxcol_.default.esc", + "ecocCollection": "enxcol_.default.ecoc", + "fields": [ + { + "keyId": { + "$binary": { + "base64": "EjRWeBI0mHYSNBI0VniQEg==", + "subType": "04" + } + }, + "path": "encryptedIndexed", + "bsonType": "string", + "queries": { + "queryType": "equality", + "contention": { + "$numberLong": "0" + } + } + }, + { + "keyId": { + "$binary": { + "base64": "q83vqxI0mHYSNBI0VniQEg==", + "subType": "04" + } + }, + "path": "encryptedUnindexed", + "bsonType": "string" + } + ] + } + } + } + }, + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "command": { + "find": "default", + "filter": { + "_id": 1 + } + }, + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "command": { + "find": "datakeys", + "filter": { + "$or": [ + { + "_id": { + "$in": [ + { + "$binary": { + "base64": "EjRWeBI0mHYSNBI0VniQEg==", + "subType": "04" + } + } + ] + } + }, + { + "keyAltNames": { + "$in": [] + } + } + ] + }, + "$db": "keyvault", + "readConcern": { + "level": "majority" + } + }, + "commandName": "find" + } + } + ] + } + ] + } + ] +} diff --git a/test/client-side-encryption/spec/unified/fle2v2-EncryptedFields-vs-EncryptedFieldsMap.json b/test/client-side-encryption/spec/unified/fle2v2-EncryptedFields-vs-EncryptedFieldsMap.json new file mode 100644 index 0000000000..b5f848c080 --- /dev/null +++ b/test/client-side-encryption/spec/unified/fle2v2-EncryptedFields-vs-EncryptedFieldsMap.json @@ -0,0 +1,256 @@ +{ + "description": "fle2v2-EncryptedFields-vs-EncryptedFieldsMap", + "schemaVersion": "1.23", + "runOnRequirements": [ + { + "minServerVersion": "7.0.0", + "serverless": "forbid", + "csfle": true, + "topologies": [ + "replicaset", + "sharded", + "load-balanced" + ] + } + ], + "createEntities": [ + { + "client": { + "id": "client0", + "autoEncryptOpts": { + "kmsProviders": { + "local": { + "key": "Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk" + } + }, + "keyVaultNamespace": "keyvault.datakeys", + "encryptedFieldsMap": { + "default.default": { + "fields": [] + } + } + }, + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "database": { + "id": "encryptedDB", + "client": "client0", + "databaseName": "default" + } + }, + { + "collection": { + "id": "encryptedColl", + "database": "encryptedDB", + "collectionName": "default" + } + } + ], + "initialData": [ + { + "databaseName": "keyvault", + "collectionName": "datakeys", + "documents": [ + { + "_id": { + "$binary": { + "base64": "q83vqxI0mHYSNBI0VniQEg==", + "subType": "04" + } + }, + "keyMaterial": { + "$binary": { + "base64": "HBk9BWihXExNDvTp1lUxOuxuZK2Pe2ZdVdlsxPEBkiO1bS4mG5NNDsQ7zVxJAH8BtdOYp72Ku4Y3nwc0BUpIKsvAKX4eYXtlhv5zUQxWdeNFhg9qK7qb8nqhnnLeT0f25jFSqzWJoT379hfwDeu0bebJHr35QrJ8myZdPMTEDYF08QYQ48ShRBli0S+QzBHHAQiM2iJNr4svg2WR8JSeWQ==", + "subType": "00" + } + }, + "creationDate": { + "$date": { + "$numberLong": "1648914851981" + } + }, + "updateDate": { + "$date": { + "$numberLong": "1648914851981" + } + }, + "status": { + "$numberInt": "0" + }, + "masterKey": { + "provider": "local" + } + } + ] + }, + { + "databaseName": "default", + "collectionName": "default", + "documents": [], + "createOptions": { + "encryptedFields": { + "fields": [ + { + "keyId": { + "$binary": { + "base64": "EjRWeBI0mHYSNBI0VniQEg==", + "subType": "04" + } + }, + "path": "encryptedIndexed", + "bsonType": "string", + "queries": { + "queryType": "equality", + "contention": { + "$numberLong": "0" + } + } + }, + { + "keyId": { + "$binary": { + "base64": "q83vqxI0mHYSNBI0VniQEg==", + "subType": "04" + } + }, + "path": "encryptedUnindexed", + "bsonType": "string" + } + ] + } + } + } + ], + "tests": [ + { + "description": "encryptedFieldsMap is preferred over remote encryptedFields", + "operations": [ + { + "object": "encryptedColl", + "name": "insertOne", + "arguments": { + "document": { + "_id": 1, + "encryptedUnindexed": { + "$binary": { + "base64": "BqvN76sSNJh2EjQSNFZ4kBICTQaVZPWgXp41I7mPV1rLFTtw1tXzjcdSEyxpKKqujlko5TeizkB9hHQ009dVY1+fgIiDcefh+eQrm3CkhQ==", + "subType": "06" + } + } + } + } + }, + { + "object": "encryptedColl", + "name": "find", + "arguments": { + "filter": { + "_id": 1 + } + }, + "expectResult": [ + { + "_id": 1, + "encryptedUnindexed": "value123" + } + ] + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "databaseName": "default", + "commandName": "insert", + "command": { + "insert": "default", + "documents": [ + { + "_id": 1, + "encryptedUnindexed": { + "$binary": { + "base64": "BqvN76sSNJh2EjQSNFZ4kBICTQaVZPWgXp41I7mPV1rLFTtw1tXzjcdSEyxpKKqujlko5TeizkB9hHQ009dVY1+fgIiDcefh+eQrm3CkhQ==", + "subType": "06" + } + } + } + ], + "ordered": true + } + } + }, + { + "commandStartedEvent": { + "databaseName": "default", + "commandName": "find", + "command": { + "find": "default", + "filter": { + "_id": 1 + } + } + } + }, + { + "commandStartedEvent": { + "databaseName": "keyvault", + "commandName": "find", + "command": { + "find": "datakeys", + "filter": { + "$or": [ + { + "_id": { + "$in": [ + { + "$binary": { + "base64": "q83vqxI0mHYSNBI0VniQEg==", + "subType": "04" + } + } + ] + } + }, + { + "keyAltNames": { + "$in": [] + } + } + ] + }, + "$db": "keyvault", + "readConcern": { + "level": "majority" + } + } + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "default", + "databaseName": "default", + "documents": [ + { + "_id": 1, + "encryptedUnindexed": { + "$binary": { + "base64": "BqvN76sSNJh2EjQSNFZ4kBICTQaVZPWgXp41I7mPV1rLFTtw1tXzjcdSEyxpKKqujlko5TeizkB9hHQ009dVY1+fgIiDcefh+eQrm3CkhQ==", + "subType": "06" + } + } + } + ] + } + ] + } + ] +} diff --git a/test/client-side-encryption/spec/unified/localSchema.json b/test/client-side-encryption/spec/unified/localSchema.json new file mode 100644 index 0000000000..aee323d949 --- /dev/null +++ b/test/client-side-encryption/spec/unified/localSchema.json @@ -0,0 +1,343 @@ +{ + "description": "localSchema", + "schemaVersion": "1.23", + "runOnRequirements": [ + { + "minServerVersion": "4.1.10", + "csfle": true + } + ], + "createEntities": [ + { + "client": { + "id": "client0", + "autoEncryptOpts": { + "schemaMap": { + "default.default": { + "properties": { + "encrypted_w_altname": { + "encrypt": { + "keyId": "/altname", + "bsonType": "string", + "algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Random" + } + }, + "encrypted_string": { + "encrypt": { + "keyId": [ + { + "$binary": { + "base64": "AAAAAAAAAAAAAAAAAAAAAA==", + "subType": "04" + } + } + ], + "bsonType": "string", + "algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic" + } + }, + "random": { + "encrypt": { + "keyId": [ + { + "$binary": { + "base64": "AAAAAAAAAAAAAAAAAAAAAA==", + "subType": "04" + } + } + ], + "bsonType": "string", + "algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Random" + } + }, + "encrypted_string_equivalent": { + "encrypt": { + "keyId": [ + { + "$binary": { + "base64": "AAAAAAAAAAAAAAAAAAAAAA==", + "subType": "04" + } + } + ], + "bsonType": "string", + "algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic" + } + } + }, + "bsonType": "object" + } + }, + "keyVaultNamespace": "keyvault.datakeys", + "kmsProviders": { + "aws": { + "accessKeyId": { + "$$placeholder": 1 + }, + "secretAccessKey": { + "$$placeholder": 1 + }, + "sessionToken": { + "$$placeholder": 1 + } + } + } + }, + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "client": { + "id": "client1", + "autoEncryptOpts": { + "schemaMap": { + "default.default": { + "properties": { + "test": { + "bsonType": "string" + } + }, + "bsonType": "object", + "required": [ + "test" + ] + } + }, + "keyVaultNamespace": "keyvault.datakeys", + "kmsProviders": { + "aws": { + "accessKeyId": { + "$$placeholder": 1 + }, + "secretAccessKey": { + "$$placeholder": 1 + }, + "sessionToken": { + "$$placeholder": 1 + } + } + } + }, + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "database": { + "id": "encryptedDB", + "client": "client0", + "databaseName": "default" + } + }, + { + "collection": { + "id": "encryptedColl", + "database": "encryptedDB", + "collectionName": "default" + } + }, + { + "database": { + "id": "encryptedDB2", + "client": "client1", + "databaseName": "default" + } + }, + { + "collection": { + "id": "encryptedColl2", + "database": "encryptedDB2", + "collectionName": "default" + } + } + ], + "initialData": [ + { + "databaseName": "keyvault", + "collectionName": "datakeys", + "documents": [ + { + "status": 1, + "_id": { + "$binary": { + "base64": "AAAAAAAAAAAAAAAAAAAAAA==", + "subType": "04" + } + }, + "masterKey": { + "provider": "aws", + "key": "arn:aws:kms:us-east-1:579766882180:key/89fcc2c4-08b0-4bd9-9f25-e30687b580d0", + "region": "us-east-1" + }, + "updateDate": { + "$date": { + "$numberLong": "1552949630483" + } + }, + "keyMaterial": { + "$binary": { + "base64": "AQICAHhQNmWG2CzOm1dq3kWLM+iDUZhEqnhJwH9wZVpuZ94A8gEqnsxXlR51T5EbEVezUqqKAAAAwjCBvwYJKoZIhvcNAQcGoIGxMIGuAgEAMIGoBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDHa4jo6yp0Z18KgbUgIBEIB74sKxWtV8/YHje5lv5THTl0HIbhSwM6EqRlmBiFFatmEWaeMk4tO4xBX65eq670I5TWPSLMzpp8ncGHMmvHqRajNBnmFtbYxN3E3/WjxmdbOOe+OXpnGJPcGsftc7cB2shRfA4lICPnE26+oVNXT6p0Lo20nY5XC7jyCO", + "subType": "00" + } + }, + "creationDate": { + "$date": { + "$numberLong": "1552949630483" + } + }, + "keyAltNames": [ + "altname", + "another_altname" + ] + } + ] + }, + { + "databaseName": "default", + "collectionName": "default", + "documents": [] + } + ], + "tests": [ + { + "description": "A local schema should override", + "operations": [ + { + "object": "encryptedColl", + "name": "insertOne", + "arguments": { + "document": { + "_id": 1, + "encrypted_string": "string0" + } + } + }, + { + "object": "encryptedColl", + "name": "find", + "arguments": { + "filter": { + "_id": 1 + } + }, + "expectResult": [ + { + "_id": 1, + "encrypted_string": "string0" + } + ] + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "databaseName": "keyvault", + "commandName": "find", + "command": { + "find": "datakeys", + "filter": { + "$or": [ + { + "_id": { + "$in": [ + { + "$binary": { + "base64": "AAAAAAAAAAAAAAAAAAAAAA==", + "subType": "04" + } + } + ] + } + }, + { + "keyAltNames": { + "$in": [] + } + } + ] + }, + "readConcern": { + "level": "majority" + } + } + } + }, + { + "commandStartedEvent": { + "commandName": "insert", + "command": { + "insert": "default", + "documents": [ + { + "_id": 1, + "encrypted_string": { + "$binary": { + "base64": "AQAAAAAAAAAAAAAAAAAAAAACwj+3zkv2VM+aTfk60RqhXq6a/77WlLwu/BxXFkL7EppGsju/m8f0x5kBDD3EZTtGALGXlym5jnpZAoSIkswHoA==", + "subType": "06" + } + } + } + ], + "ordered": true + } + } + }, + { + "commandStartedEvent": { + "commandName": "find", + "command": { + "find": "default", + "filter": { + "_id": 1 + } + } + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "default", + "databaseName": "default", + "documents": [ + { + "_id": 1, + "encrypted_string": { + "$binary": { + "base64": "AQAAAAAAAAAAAAAAAAAAAAACwj+3zkv2VM+aTfk60RqhXq6a/77WlLwu/BxXFkL7EppGsju/m8f0x5kBDD3EZTtGALGXlym5jnpZAoSIkswHoA==", + "subType": "06" + } + } + } + ] + } + ] + }, + { + "description": "A local schema with no encryption is an error", + "operations": [ + { + "object": "encryptedColl2", + "name": "insertOne", + "arguments": { + "document": { + "_id": 1, + "encrypted_string": "string0" + } + }, + "expectError": { + "isError": true, + "errorContains": "JSON schema keyword 'required' is only allowed with a remote schema" + } + } + ] + } + ] +} diff --git a/test/client-side-encryption/spec/unified/maxWireVersion.json b/test/client-side-encryption/spec/unified/maxWireVersion.json new file mode 100644 index 0000000000..d0af75ac99 --- /dev/null +++ b/test/client-side-encryption/spec/unified/maxWireVersion.json @@ -0,0 +1,101 @@ +{ + "description": "maxWireVersion", + "schemaVersion": "1.23", + "runOnRequirements": [ + { + "maxServerVersion": "4.0.99", + "csfle": true + } + ], + "createEntities": [ + { + "client": { + "id": "client0", + "autoEncryptOpts": { + "kmsProviders": { + "aws": {} + }, + "keyVaultNamespace": "keyvault.datakeys", + "extraOptions": { + "mongocryptdBypassSpawn": true + } + } + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "default" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "default" + } + } + ], + "initialData": [ + { + "databaseName": "keyvault", + "collectionName": "datakeys", + "documents": [ + { + "status": 1, + "_id": { + "$binary": { + "base64": "AAAAAAAAAAAAAAAAAAAAAA==", + "subType": "04" + } + }, + "masterKey": { + "provider": "aws", + "key": "arn:aws:kms:us-east-1:579766882180:key/89fcc2c4-08b0-4bd9-9f25-e30687b580d0", + "region": "us-east-1" + }, + "updateDate": { + "$date": { + "$numberLong": "1552949630483" + } + }, + "keyMaterial": { + "$binary": { + "base64": "AQICAHhQNmWG2CzOm1dq3kWLM+iDUZhEqnhJwH9wZVpuZ94A8gEqnsxXlR51T5EbEVezUqqKAAAAwjCBvwYJKoZIhvcNAQcGoIGxMIGuAgEAMIGoBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDHa4jo6yp0Z18KgbUgIBEIB74sKxWtV8/YHje5lv5THTl0HIbhSwM6EqRlmBiFFatmEWaeMk4tO4xBX65eq670I5TWPSLMzpp8ncGHMmvHqRajNBnmFtbYxN3E3/WjxmdbOOe+OXpnGJPcGsftc7cB2shRfA4lICPnE26+oVNXT6p0Lo20nY5XC7jyCO", + "subType": "00" + } + }, + "creationDate": { + "$date": { + "$numberLong": "1552949630483" + } + }, + "keyAltNames": [ + "altname", + "another_altname" + ] + } + ] + } + ], + "tests": [ + { + "description": "operation fails with maxWireVersion < 8", + "operations": [ + { + "name": "insertOne", + "object": "collection0", + "arguments": { + "document": { + "encrypted_string": "string0" + } + }, + "expectError": { + "errorContains": "Auto-encryption requires a minimum MongoDB version of 4.2" + } + } + ] + } + ] +} diff --git a/test/helpers.py b/test/helpers.py index 22bdc0d25d..163bf01c12 100644 --- a/test/helpers.py +++ b/test/helpers.py @@ -12,137 +12,22 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Shared constants and helper methods for pymongo, bson, and gridfs test suites.""" +"""Shared helper methods for pymongo, bson, and gridfs test suites.""" from __future__ import annotations import asyncio -import base64 -import gc -import multiprocessing -import os -import signal -import socket -import subprocess -import sys import threading -import time import traceback -import unittest -import warnings -from inspect import iscoroutinefunction - -from pymongo._asyncio_task import create_task - -try: - import ipaddress - - HAVE_IPADDRESS = True -except ImportError: - HAVE_IPADDRESS = False from functools import wraps -from typing import Any, Callable, Dict, Generator, Optional, no_type_check -from unittest import SkipTest +from typing import Optional, no_type_check -from bson.son import SON -from pymongo import common, message +from bson import SON +from pymongo import common +from pymongo._asyncio_task import create_task from pymongo.read_preferences import ReadPreference -from pymongo.ssl_support import HAVE_SSL, _ssl # type:ignore[attr-defined] -from pymongo.synchronous.uri_parser import parse_uri - -if HAVE_SSL: - import ssl _IS_SYNC = True -# Enable debug output for uncollectable objects. PyPy does not have set_debug. -if hasattr(gc, "set_debug"): - gc.set_debug( - gc.DEBUG_UNCOLLECTABLE | getattr(gc, "DEBUG_OBJECTS", 0) | getattr(gc, "DEBUG_INSTANCES", 0) - ) - -# The host and port of a single mongod or mongos, or the seed host -# for a replica set. -host = os.environ.get("DB_IP", "localhost") -port = int(os.environ.get("DB_PORT", 27017)) -IS_SRV = "mongodb+srv" in host - -db_user = os.environ.get("DB_USER", "user") -db_pwd = os.environ.get("DB_PASSWORD", "password") - -CERT_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), "certificates") -CLIENT_PEM = os.environ.get("CLIENT_PEM", os.path.join(CERT_PATH, "client.pem")) -CA_PEM = os.environ.get("CA_PEM", os.path.join(CERT_PATH, "ca.pem")) - -TLS_OPTIONS: Dict = {"tls": True} -if CLIENT_PEM: - TLS_OPTIONS["tlsCertificateKeyFile"] = CLIENT_PEM -if CA_PEM: - TLS_OPTIONS["tlsCAFile"] = CA_PEM - -COMPRESSORS = os.environ.get("COMPRESSORS") -MONGODB_API_VERSION = os.environ.get("MONGODB_API_VERSION") -TEST_LOADBALANCER = bool(os.environ.get("TEST_LOAD_BALANCER")) -SINGLE_MONGOS_LB_URI = os.environ.get("SINGLE_MONGOS_LB_URI") -MULTI_MONGOS_LB_URI = os.environ.get("MULTI_MONGOS_LB_URI") - -if TEST_LOADBALANCER: - res = parse_uri(SINGLE_MONGOS_LB_URI or "") - host, port = res["nodelist"][0] - db_user = res["username"] or db_user - db_pwd = res["password"] or db_pwd - - -# Shared KMS data. -LOCAL_MASTER_KEY = base64.b64decode( - b"Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ" - b"5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk" -) -AWS_CREDS = { - "accessKeyId": os.environ.get("FLE_AWS_KEY", ""), - "secretAccessKey": os.environ.get("FLE_AWS_SECRET", ""), -} -AWS_CREDS_2 = { - "accessKeyId": os.environ.get("FLE_AWS_KEY2", ""), - "secretAccessKey": os.environ.get("FLE_AWS_SECRET2", ""), -} -AZURE_CREDS = { - "tenantId": os.environ.get("FLE_AZURE_TENANTID", ""), - "clientId": os.environ.get("FLE_AZURE_CLIENTID", ""), - "clientSecret": os.environ.get("FLE_AZURE_CLIENTSECRET", ""), -} -GCP_CREDS = { - "email": os.environ.get("FLE_GCP_EMAIL", ""), - "privateKey": os.environ.get("FLE_GCP_PRIVATEKEY", ""), -} -KMIP_CREDS = {"endpoint": os.environ.get("FLE_KMIP_ENDPOINT", "localhost:5698")} - -# Ensure Evergreen metadata doesn't result in truncation -os.environ.setdefault("MONGOB_LOG_MAX_DOCUMENT_LENGTH", "2000") - - -def is_server_resolvable(): - """Returns True if 'server' is resolvable.""" - socket_timeout = socket.getdefaulttimeout() - socket.setdefaulttimeout(1) - try: - try: - socket.gethostbyname("server") - return True - except OSError: - return False - finally: - socket.setdefaulttimeout(socket_timeout) - - -def _create_user(authdb, user, pwd=None, roles=None, **kwargs): - cmd = SON([("createUser", user)]) - # X509 doesn't use a password - if pwd: - cmd["pwd"] = pwd - cmd["roles"] = roles or ["root"] - cmd.update(**kwargs) - return authdb.command(cmd) - def repl_set_step_down(client, **kwargs): """Run replSetStepDown, first unfreezing a secondary with replSetFreeze.""" @@ -237,133 +122,10 @@ def __del__(self): raise Exception(msg) -def _all_users(db): - return {u["user"] for u in db.command("usersInfo").get("users", [])} - - -def sanitize_cmd(cmd): - cp = cmd.copy() - cp.pop("$clusterTime", None) - cp.pop("$db", None) - cp.pop("$readPreference", None) - cp.pop("lsid", None) - if MONGODB_API_VERSION: - # Stable API parameters - cp.pop("apiVersion", None) - # OP_MSG encoding may move the payload type one field to the - # end of the command. Do the same here. - name = next(iter(cp)) - try: - identifier = message._FIELD_MAP[name] - docs = cp.pop(identifier) - cp[identifier] = docs - except KeyError: - pass - return cp - - -def sanitize_reply(reply): - cp = reply.copy() - cp.pop("$clusterTime", None) - cp.pop("operationTime", None) - return cp - - -def print_thread_tracebacks() -> None: - """Print all Python thread tracebacks.""" - for thread_id, frame in sys._current_frames().items(): - sys.stderr.write(f"\n--- Traceback for thread {thread_id} ---\n") - traceback.print_stack(frame, file=sys.stderr) - - -def print_thread_stacks(pid: int) -> None: - """Print all C-level thread stacks for a given process id.""" - if sys.platform == "darwin": - cmd = ["lldb", "--attach-pid", f"{pid}", "--batch", "--one-line", '"thread backtrace all"'] - else: - cmd = ["gdb", f"--pid={pid}", "--batch", '--eval-command="thread apply all bt"'] - - try: - res = subprocess.run( - cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, encoding="utf-8" - ) - except Exception as exc: - sys.stderr.write(f"Could not print C-level thread stacks because {cmd[0]} failed: {exc}") - else: - sys.stderr.write(res.stdout) - - # Global knobs to speed up the test suite. global_knobs = client_knobs(events_queue_frequency=0.05) -def _get_executors(topology): - executors = [] - for server in topology._servers.values(): - # Some MockMonitor do not have an _executor. - if hasattr(server._monitor, "_executor"): - executors.append(server._monitor._executor) - if hasattr(server._monitor, "_rtt_monitor"): - executors.append(server._monitor._rtt_monitor._executor) - executors.append(topology._Topology__events_executor) - if topology._srv_monitor: - executors.append(topology._srv_monitor._executor) - - return [e for e in executors if e is not None] - - -def print_running_topology(topology): - running = [e for e in _get_executors(topology) if not e._stopped] - if running: - print( - "WARNING: found Topology with running threads:\n" - f" Threads: {running}\n" - f" Topology: {topology}\n" - f" Creation traceback:\n{topology._settings._stack}" - ) - - -def test_cases(suite): - """Iterator over all TestCases within a TestSuite.""" - for suite_or_case in suite._tests: - if isinstance(suite_or_case, unittest.TestCase): - # unittest.TestCase - yield suite_or_case - else: - # unittest.TestSuite - yield from test_cases(suite_or_case) - - -# Helper method to workaround https://bugs.python.org/issue21724 -def clear_warning_registry(): - """Clear the __warningregistry__ for all modules.""" - for _, module in list(sys.modules.items()): - if hasattr(module, "__warningregistry__"): - module.__warningregistry__ = {} # type:ignore[attr-defined] - - -class SystemCertsPatcher: - def __init__(self, ca_certs): - if ( - ssl.OPENSSL_VERSION.lower().startswith("libressl") - and sys.platform == "darwin" - and not _ssl.IS_PYOPENSSL - ): - raise SkipTest( - "LibreSSL on OSX doesn't support setting CA certificates " - "using SSL_CERT_FILE environment variable." - ) - self.original_certs = os.environ.get("SSL_CERT_FILE") - # Tell OpenSSL where CA certificates live. - os.environ["SSL_CERT_FILE"] = ca_certs - - def disable(self): - if self.original_certs is None: - os.environ.pop("SSL_CERT_FILE") - else: - os.environ["SSL_CERT_FILE"] = self.original_certs - - if _IS_SYNC: PARENT = threading.Thread else: diff --git a/test/helpers_shared.py b/test/helpers_shared.py new file mode 100644 index 0000000000..49cf131808 --- /dev/null +++ b/test/helpers_shared.py @@ -0,0 +1,271 @@ +# Copyright 2019-present MongoDB, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import base64 +import gc +import os +import socket +import subprocess +import sys +import traceback +import unittest +from pathlib import Path + +try: + import ipaddress + + HAVE_IPADDRESS = True +except ImportError: + HAVE_IPADDRESS = False +from functools import wraps +from typing import no_type_check +from unittest import SkipTest + +from bson.son import SON +from pymongo import message +from pymongo.ssl_support import HAVE_SSL, _ssl # type:ignore[attr-defined] +from pymongo.synchronous.uri_parser import parse_uri + +if HAVE_SSL: + import ssl + + +# Enable debug output for uncollectable objects. PyPy does not have set_debug. +if hasattr(gc, "set_debug"): + gc.set_debug( + gc.DEBUG_UNCOLLECTABLE | getattr(gc, "DEBUG_OBJECTS", 0) | getattr(gc, "DEBUG_INSTANCES", 0) + ) + +# The host and port of a single mongod or mongos, or the seed host +# for a replica set. +host = os.environ.get("DB_IP", "localhost") +port = int(os.environ.get("DB_PORT", 27017)) +IS_SRV = "mongodb+srv" in host + +db_user = os.environ.get("DB_USER", "user") +db_pwd = os.environ.get("DB_PASSWORD", "password") + +HERE = Path(__file__).absolute() +CERT_PATH = str(HERE.parent / "certificates") +CLIENT_PEM = os.environ.get("CLIENT_PEM", os.path.join(CERT_PATH, "client.pem")) +CA_PEM = os.environ.get("CA_PEM", os.path.join(CERT_PATH, "ca.pem")) + +TLS_OPTIONS: dict = {"tls": True} +if CLIENT_PEM: + TLS_OPTIONS["tlsCertificateKeyFile"] = CLIENT_PEM +if CA_PEM: + TLS_OPTIONS["tlsCAFile"] = CA_PEM + +COMPRESSORS = os.environ.get("COMPRESSORS") +MONGODB_API_VERSION = os.environ.get("MONGODB_API_VERSION") +TEST_LOADBALANCER = bool(os.environ.get("TEST_LOAD_BALANCER")) +SINGLE_MONGOS_LB_URI = os.environ.get("SINGLE_MONGOS_LB_URI") +MULTI_MONGOS_LB_URI = os.environ.get("MULTI_MONGOS_LB_URI") + +if TEST_LOADBALANCER: + res = parse_uri(SINGLE_MONGOS_LB_URI or "") + host, port = res["nodelist"][0] + db_user = res["username"] or db_user + db_pwd = res["password"] or db_pwd + + +# Shared KMS data. +LOCAL_MASTER_KEY = base64.b64decode( + b"Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ" + b"5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk" +) +AWS_CREDS = { + "accessKeyId": os.environ.get("FLE_AWS_KEY", ""), + "secretAccessKey": os.environ.get("FLE_AWS_SECRET", ""), +} +AWS_CREDS_2 = { + "accessKeyId": os.environ.get("FLE_AWS_KEY2", ""), + "secretAccessKey": os.environ.get("FLE_AWS_SECRET2", ""), +} +AZURE_CREDS = { + "tenantId": os.environ.get("FLE_AZURE_TENANTID", ""), + "clientId": os.environ.get("FLE_AZURE_CLIENTID", ""), + "clientSecret": os.environ.get("FLE_AZURE_CLIENTSECRET", ""), +} +GCP_CREDS = { + "email": os.environ.get("FLE_GCP_EMAIL", ""), + "privateKey": os.environ.get("FLE_GCP_PRIVATEKEY", ""), +} +KMIP_CREDS = {"endpoint": os.environ.get("FLE_KMIP_ENDPOINT", "localhost:5698")} +AWS_TEMP_CREDS = { + "accessKeyId": os.environ.get("CSFLE_AWS_TEMP_ACCESS_KEY_ID", ""), + "secretAccessKey": os.environ.get("CSFLE_AWS_TEMP_SECRET_ACCESS_KEY", ""), + "sessionToken": os.environ.get("CSFLE_AWS_TEMP_SESSION_TOKEN", ""), +} + +ALL_KMS_PROVIDERS = dict( + aws=AWS_CREDS, + azure=AZURE_CREDS, + gcp=GCP_CREDS, + local=dict(key=LOCAL_MASTER_KEY), + kmip=KMIP_CREDS, +) +DEFAULT_KMS_TLS = dict(kmip=dict(tlsCAFile=CA_PEM, tlsCertificateKeyFile=CLIENT_PEM)) + +# Ensure Evergreen metadata doesn't result in truncation +os.environ.setdefault("MONGOB_LOG_MAX_DOCUMENT_LENGTH", "2000") + + +def is_server_resolvable(): + """Returns True if 'server' is resolvable.""" + socket_timeout = socket.getdefaulttimeout() + socket.setdefaulttimeout(1) + try: + try: + socket.gethostbyname("server") + return True + except OSError: + return False + finally: + socket.setdefaulttimeout(socket_timeout) + + +def _create_user(authdb, user, pwd=None, roles=None, **kwargs): + cmd = SON([("createUser", user)]) + # X509 doesn't use a password + if pwd: + cmd["pwd"] = pwd + cmd["roles"] = roles or ["root"] + cmd.update(**kwargs) + return authdb.command(cmd) + + +def _all_users(db): + return {u["user"] for u in db.command("usersInfo").get("users", [])} + + +def sanitize_cmd(cmd): + cp = cmd.copy() + cp.pop("$clusterTime", None) + cp.pop("$db", None) + cp.pop("$readPreference", None) + cp.pop("lsid", None) + if MONGODB_API_VERSION: + # Stable API parameters + cp.pop("apiVersion", None) + # OP_MSG encoding may move the payload type one field to the + # end of the command. Do the same here. + name = next(iter(cp)) + try: + identifier = message._FIELD_MAP[name] + docs = cp.pop(identifier) + cp[identifier] = docs + except KeyError: + pass + return cp + + +def sanitize_reply(reply): + cp = reply.copy() + cp.pop("$clusterTime", None) + cp.pop("operationTime", None) + return cp + + +def print_thread_tracebacks() -> None: + """Print all Python thread tracebacks.""" + for thread_id, frame in sys._current_frames().items(): + sys.stderr.write(f"\n--- Traceback for thread {thread_id} ---\n") + traceback.print_stack(frame, file=sys.stderr) + + +def print_thread_stacks(pid: int) -> None: + """Print all C-level thread stacks for a given process id.""" + if sys.platform == "darwin": + cmd = ["lldb", "--attach-pid", f"{pid}", "--batch", "--one-line", '"thread backtrace all"'] + else: + cmd = ["gdb", f"--pid={pid}", "--batch", '--eval-command="thread apply all bt"'] + + try: + res = subprocess.run( + cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, encoding="utf-8" + ) + except Exception as exc: + sys.stderr.write(f"Could not print C-level thread stacks because {cmd[0]} failed: {exc}") + else: + sys.stderr.write(res.stdout) + + +def _get_executors(topology): + executors = [] + for server in topology._servers.values(): + # Some MockMonitor do not have an _executor. + if hasattr(server._monitor, "_executor"): + executors.append(server._monitor._executor) + if hasattr(server._monitor, "_rtt_monitor"): + executors.append(server._monitor._rtt_monitor._executor) + executors.append(topology._Topology__events_executor) + if topology._srv_monitor: + executors.append(topology._srv_monitor._executor) + + return [e for e in executors if e is not None] + + +def print_running_topology(topology): + running = [e for e in _get_executors(topology) if not e._stopped] + if running: + print( + "WARNING: found Topology with running threads:\n" + f" Threads: {running}\n" + f" Topology: {topology}\n" + f" Creation traceback:\n{topology._settings._stack}" + ) + + +def test_cases(suite): + """Iterator over all TestCases within a TestSuite.""" + for suite_or_case in suite._tests: + if isinstance(suite_or_case, unittest.TestCase): + # unittest.TestCase + yield suite_or_case + else: + # unittest.TestSuite + yield from test_cases(suite_or_case) + + +# Helper method to workaround https://bugs.python.org/issue21724 +def clear_warning_registry(): + """Clear the __warningregistry__ for all modules.""" + for _, module in list(sys.modules.items()): + if hasattr(module, "__warningregistry__"): + module.__warningregistry__ = {} # type:ignore[attr-defined] + + +class SystemCertsPatcher: + def __init__(self, ca_certs): + if ( + ssl.OPENSSL_VERSION.lower().startswith("libressl") + and sys.platform == "darwin" + and not _ssl.IS_PYOPENSSL + ): + raise SkipTest( + "LibreSSL on OSX doesn't support setting CA certificates " + "using SSL_CERT_FILE environment variable." + ) + self.original_certs = os.environ.get("SSL_CERT_FILE") + # Tell OpenSSL where CA certificates live. + os.environ["SSL_CERT_FILE"] = ca_certs + + def disable(self): + if self.original_certs is None: + os.environ.pop("SSL_CERT_FILE") + else: + os.environ["SSL_CERT_FILE"] = self.original_certs diff --git a/test/test_encryption.py b/test/test_encryption.py index 5c8813203d..46d8c785c4 100644 --- a/test/test_encryption.py +++ b/test/test_encryption.py @@ -54,11 +54,14 @@ from test import ( unittest, ) -from test.helpers import ( +from test.helpers_shared import ( + ALL_KMS_PROVIDERS, AWS_CREDS, + AWS_TEMP_CREDS, AZURE_CREDS, CA_PEM, CLIENT_PEM, + DEFAULT_KMS_TLS, GCP_CREDS, KMIP_CREDS, LOCAL_MASTER_KEY, @@ -204,7 +207,7 @@ def test_init_kms_tls_options(self): opts = AutoEncryptionOpts( {}, "k.d", - kms_tls_options={"kmip": {"tlsCAFile": CA_PEM, "tlsCertificateKeyFile": CLIENT_PEM}}, + kms_tls_options=DEFAULT_KMS_TLS, ) _kms_ssl_contexts = _parse_kms_tls_options(opts._kms_tls_options, _IS_SYNC) ctx = _kms_ssl_contexts["kmip"] @@ -614,17 +617,10 @@ def test_with_statement(self): # Spec tests -AWS_TEMP_CREDS = { - "accessKeyId": os.environ.get("CSFLE_AWS_TEMP_ACCESS_KEY_ID", ""), - "secretAccessKey": os.environ.get("CSFLE_AWS_TEMP_SECRET_ACCESS_KEY", ""), - "sessionToken": os.environ.get("CSFLE_AWS_TEMP_SESSION_TOKEN", ""), -} - AWS_TEMP_NO_SESSION_CREDS = { "accessKeyId": os.environ.get("CSFLE_AWS_TEMP_ACCESS_KEY_ID", ""), "secretAccessKey": os.environ.get("CSFLE_AWS_TEMP_SECRET_ACCESS_KEY", ""), } -KMS_TLS_OPTS = {"kmip": {"tlsCAFile": CA_PEM, "tlsCertificateKeyFile": CLIENT_PEM}} class TestSpec(SpecRunner): @@ -661,7 +657,7 @@ def parse_auto_encrypt_opts(self, opts): self.skipTest("GCP environment credentials are not set") if "kmip" in kms_providers: kms_providers["kmip"] = KMIP_CREDS - opts["kms_tls_options"] = KMS_TLS_OPTS + opts["kms_tls_options"] = DEFAULT_KMS_TLS if "key_vault_namespace" not in opts: opts["key_vault_namespace"] = "keyvault.datakeys" if "extra_options" in opts: @@ -755,14 +751,6 @@ def run_scenario(self): ) # Prose Tests -ALL_KMS_PROVIDERS = { - "aws": AWS_CREDS, - "azure": AZURE_CREDS, - "gcp": GCP_CREDS, - "kmip": KMIP_CREDS, - "local": {"key": LOCAL_MASTER_KEY}, -} - LOCAL_KEY_ID = Binary(base64.b64decode(b"LOCALAAAAAAAAAAAAAAAAA=="), UUID_SUBTYPE) AWS_KEY_ID = Binary(base64.b64decode(b"AWSAAAAAAAAAAAAAAAAAAA=="), UUID_SUBTYPE) AZURE_KEY_ID = Binary(base64.b64decode(b"AZUREAAAAAAAAAAAAAAAAA=="), UUID_SUBTYPE) @@ -849,13 +837,17 @@ def setUp(self): self.KMS_PROVIDERS, "keyvault.datakeys", schema_map=schemas, - kms_tls_options=KMS_TLS_OPTS, + kms_tls_options=DEFAULT_KMS_TLS, ) self.client_encrypted = self.rs_or_single_client( auto_encryption_opts=opts, uuidRepresentation="standard" ) self.client_encryption = self.create_client_encryption( - self.KMS_PROVIDERS, "keyvault.datakeys", self.client, OPTS, kms_tls_options=KMS_TLS_OPTS + self.KMS_PROVIDERS, + "keyvault.datakeys", + self.client, + OPTS, + kms_tls_options=DEFAULT_KMS_TLS, ) self.listener.reset() @@ -1062,7 +1054,7 @@ def _test_corpus(self, opts): "keyvault.datakeys", client_context.client, OPTS, - kms_tls_options=KMS_TLS_OPTS, + kms_tls_options=DEFAULT_KMS_TLS, ) corpus = self.fix_up_curpus(json_data("corpus", "corpus.json")) @@ -1154,7 +1146,7 @@ def _test_corpus(self, opts): def test_corpus(self): opts = AutoEncryptionOpts( - self.kms_providers(), "keyvault.datakeys", kms_tls_options=KMS_TLS_OPTS + self.kms_providers(), "keyvault.datakeys", kms_tls_options=DEFAULT_KMS_TLS ) self._test_corpus(opts) @@ -1165,7 +1157,7 @@ def test_corpus_local_schema(self): self.kms_providers(), "keyvault.datakeys", schema_map=schemas, - kms_tls_options=KMS_TLS_OPTS, + kms_tls_options=DEFAULT_KMS_TLS, ) self._test_corpus(opts) @@ -1296,7 +1288,7 @@ def setUp(self): key_vault_namespace="keyvault.datakeys", key_vault_client=client_context.client, codec_options=OPTS, - kms_tls_options=KMS_TLS_OPTS, + kms_tls_options=DEFAULT_KMS_TLS, ) kms_providers_invalid = copy.deepcopy(kms_providers) @@ -1308,7 +1300,7 @@ def setUp(self): key_vault_namespace="keyvault.datakeys", key_vault_client=client_context.client, codec_options=OPTS, - kms_tls_options=KMS_TLS_OPTS, + kms_tls_options=DEFAULT_KMS_TLS, ) self._kmip_host_error = None self._invalid_host_error = None @@ -2736,7 +2728,7 @@ def run_test(self, src_provider, dst_provider): key_vault_client=self.client, key_vault_namespace="keyvault.datakeys", kms_providers=ALL_KMS_PROVIDERS, - kms_tls_options=KMS_TLS_OPTS, + kms_tls_options=DEFAULT_KMS_TLS, codec_options=OPTS, ) @@ -2756,7 +2748,7 @@ def run_test(self, src_provider, dst_provider): key_vault_client=client2, key_vault_namespace="keyvault.datakeys", kms_providers=ALL_KMS_PROVIDERS, - kms_tls_options=KMS_TLS_OPTS, + kms_tls_options=DEFAULT_KMS_TLS, codec_options=OPTS, ) diff --git a/test/test_uri_spec.py b/test/test_uri_spec.py index 8f673cff4c..3d8f7b2b75 100644 --- a/test/test_uri_spec.py +++ b/test/test_uri_spec.py @@ -25,7 +25,7 @@ sys.path[0:0] = [""] from test import unittest -from test.helpers import clear_warning_registry +from test.helpers_shared import clear_warning_registry from pymongo.common import INTERNAL_URI_OPTION_NAME_MAP, _CaseInsensitiveDictionary, validate from pymongo.compression_support import _have_snappy diff --git a/test/unified-test-format/valid-pass/poc-queryable-encryption.json b/test/unified-test-format/valid-pass/poc-queryable-encryption.json new file mode 100644 index 0000000000..309d1d3b4b --- /dev/null +++ b/test/unified-test-format/valid-pass/poc-queryable-encryption.json @@ -0,0 +1,193 @@ +{ + "description": "poc-queryable-encryption", + "schemaVersion": "1.23", + "runOnRequirements": [ + { + "minServerVersion": "7.0", + "csfle": true, + "topologies": [ + "replicaset", + "load-balanced", + "sharded" + ] + } + ], + "createEntities": [ + { + "client": { + "id": "client0", + "autoEncryptOpts": { + "keyVaultNamespace": "keyvault.datakeys", + "kmsProviders": { + "local": { + "key": "Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk" + } + } + } + } + }, + { + "database": { + "id": "encryptedDB", + "client": "client0", + "databaseName": "poc-queryable-encryption" + } + }, + { + "collection": { + "id": "encryptedColl", + "database": "encryptedDB", + "collectionName": "encrypted" + } + }, + { + "client": { + "id": "client1" + } + }, + { + "database": { + "id": "unencryptedDB", + "client": "client1", + "databaseName": "poc-queryable-encryption" + } + }, + { + "collection": { + "id": "unencryptedColl", + "database": "unencryptedDB", + "collectionName": "encrypted" + } + } + ], + "initialData": [ + { + "databaseName": "keyvault", + "collectionName": "datakeys", + "documents": [ + { + "_id": { + "$binary": { + "base64": "EjRWeBI0mHYSNBI0VniQEg==", + "subType": "04" + } + }, + "keyMaterial": { + "$binary": { + "base64": "sHe0kz57YW7v8g9VP9sf/+K1ex4JqKc5rf/URX3n3p8XdZ6+15uXPaSayC6adWbNxkFskuMCOifDoTT+rkqMtFkDclOy884RuGGtUysq3X7zkAWYTKi8QAfKkajvVbZl2y23UqgVasdQu3OVBQCrH/xY00nNAs/52e958nVjBuzQkSb1T8pKJAyjZsHJ60+FtnfafDZSTAIBJYn7UWBCwQ==", + "subType": "00" + } + }, + "creationDate": { + "$date": { + "$numberLong": "1641024000000" + } + }, + "updateDate": { + "$date": { + "$numberLong": "1641024000000" + } + }, + "status": 1, + "masterKey": { + "provider": "local" + } + } + ] + }, + { + "databaseName": "poc-queryable-encryption", + "collectionName": "encrypted", + "documents": [], + "createOptions": { + "encryptedFields": { + "fields": [ + { + "keyId": { + "$binary": { + "base64": "EjRWeBI0mHYSNBI0VniQEg==", + "subType": "04" + } + }, + "path": "encryptedInt", + "bsonType": "int", + "queries": { + "queryType": "equality", + "contention": { + "$numberLong": "0" + } + } + } + ] + } + } + } + ], + "tests": [ + { + "description": "insert, replace, and find with queryable encryption", + "operations": [ + { + "object": "encryptedColl", + "name": "insertOne", + "arguments": { + "document": { + "_id": 1, + "encryptedInt": 11 + } + } + }, + { + "object": "encryptedColl", + "name": "replaceOne", + "arguments": { + "filter": { + "encryptedInt": 11 + }, + "replacement": { + "encryptedInt": 22 + } + } + }, + { + "object": "encryptedColl", + "name": "find", + "arguments": { + "filter": { + "encryptedInt": 22 + } + }, + "expectResult": [ + { + "_id": 1, + "encryptedInt": 22 + } + ] + }, + { + "object": "unencryptedColl", + "name": "find", + "arguments": { + "filter": {} + }, + "expectResult": [ + { + "_id": 1, + "encryptedInt": { + "$$type": "binData" + }, + "__safeContent__": [ + { + "$binary": { + "base64": "rhS16TJojgDDBtbluxBokvcotP1mQTGeYpNt8xd3MJQ=", + "subType": "00" + } + } + ] + } + ] + } + ] + } + ] +} diff --git a/test/unified_format.py b/test/unified_format.py index 3496b2ad44..2cbc581aca 100644 --- a/test/unified_format.py +++ b/test/unified_format.py @@ -35,6 +35,7 @@ client_knobs, unittest, ) +from test.helpers_shared import ALL_KMS_PROVIDERS, DEFAULT_KMS_TLS from test.unified_format_shared import ( KMS_TLS_OPTS, PLACEHOLDER_MAP, @@ -60,6 +61,8 @@ from test.version import Version from typing import Any, Dict, List, Mapping, Optional +import pytest + import pymongo from bson import SON, json_util from bson.codec_options import DEFAULT_CODEC_OPTIONS @@ -68,7 +71,7 @@ from gridfs.errors import CorruptGridFile from pymongo import ASCENDING, CursorType, MongoClient, _csot from pymongo.driver_info import DriverInfo -from pymongo.encryption_options import _HAVE_PYMONGOCRYPT +from pymongo.encryption_options import _HAVE_PYMONGOCRYPT, AutoEncryptionOpts from pymongo.errors import ( AutoReconnect, BulkWriteError, @@ -258,6 +261,23 @@ def _create_entity(self, entity_spec, uri=None): kwargs: dict = {} observe_events = spec.get("observeEvents", []) + if "autoEncryptOpts" in spec: + auto_encrypt_opts = spec["autoEncryptOpts"].copy() + auto_encrypt_kwargs: dict = dict(kms_tls_options=DEFAULT_KMS_TLS) + kms_providers = ALL_KMS_PROVIDERS.copy() + key_vault_namespace = auto_encrypt_opts.pop("keyVaultNamespace") + for provider_name, provider_value in auto_encrypt_opts.pop("kmsProviders").items(): + kms_providers[provider_name].update(provider_value) + extra_opts = auto_encrypt_opts.pop("extraOptions", {}) + for key, value in extra_opts.items(): + auto_encrypt_kwargs[camel_to_snake(key)] = value + for key, value in auto_encrypt_opts.items(): + auto_encrypt_kwargs[camel_to_snake(key)] = value + auto_encryption_opts = AutoEncryptionOpts( + kms_providers, key_vault_namespace, **auto_encrypt_kwargs + ) + kwargs["auto_encryption_opts"] = auto_encryption_opts + # The unified tests use topologyOpeningEvent, we use topologyOpenedEvent for i in range(len(observe_events)): if "topologyOpeningEvent" == observe_events[i]: @@ -429,7 +449,7 @@ class UnifiedSpecTestMixinV1(IntegrationTest): a class attribute ``TEST_SPEC``. """ - SCHEMA_VERSION = Version.from_string("1.22") + SCHEMA_VERSION = Version.from_string("1.23") RUN_ON_LOAD_BALANCER = True TEST_SPEC: Any TEST_PATH = "" # This gets filled in by generate_test_classes @@ -461,6 +481,13 @@ def insert_initial_data(self, initial_data): wc = WriteConcern(w="majority") else: wc = WriteConcern(w=1) + + # Remove any encryption collections associated with the collection. + collections = db.list_collection_names() + for collection in collections: + if collection in [f"enxcol_.{coll_name}.esc", f"enxcol_.{coll_name}.ecoc"]: + db.drop_collection(collection) + if documents: if opts: db.create_collection(coll_name, **opts) @@ -1501,7 +1528,14 @@ class SpecTestBase(with_metaclass(UnifiedSpecTestMeta)): # type: ignore TEST_SPEC = test_spec EXPECTED_FAILURES = expected_failures - return SpecTestBase + base = SpecTestBase + + # Add "encryption" marker if the "csfle" runOnRequirement is set. + for req in test_spec.get("runOnRequirements", []): + if req.get("csfle", False): + base = pytest.mark.encryption(base) + + return base for dirpath, _, filenames in os.walk(test_path): dirname = os.path.split(dirpath)[-1] diff --git a/test/unified_format_shared.py b/test/unified_format_shared.py index 17dd73ec8c..96b037976b 100644 --- a/test/unified_format_shared.py +++ b/test/unified_format_shared.py @@ -25,9 +25,10 @@ import time import types from collections import abc -from test.helpers import ( +from test.helpers_shared import ( AWS_CREDS, AWS_CREDS_2, + AWS_TEMP_CREDS, AZURE_CREDS, CA_PEM, CLIENT_PEM, @@ -118,10 +119,22 @@ ("kmip", KMIP_CREDS), ("kmip:name1", KMIP_CREDS), ]: + # Use the temp aws creds for autoEncryptOpts. + if provider_name == "aws": + for key, value in AWS_TEMP_CREDS.items(): + placeholder = f"/autoEncryptOpts/kmsProviders/{provider_name}/{key}" + PLACEHOLDER_MAP[placeholder] = value + for key, value in provider_data.items(): placeholder = f"/clientEncryptionOpts/kmsProviders/{provider_name}/{key}" PLACEHOLDER_MAP[placeholder] = value + if provider_name == "aws": + continue + + placeholder = f"/autoEncryptOpts/kmsProviders/{provider_name}/{key}" + PLACEHOLDER_MAP[placeholder] = value + OIDC_ENV = os.environ.get("OIDC_ENV", "test") if OIDC_ENV == "test": PLACEHOLDER_MAP["/uriOptions/authMechanismProperties"] = {"ENVIRONMENT": "test"} From 9a9a65c6170ac202d618593a7e50165a85dcb1b9 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 20 Aug 2025 18:42:06 -0500 Subject: [PATCH 05/53] PYTHON-5496 Update CSOT tests for change in dropIndex behavior in 8.3 (#2498) --- test/asynchronous/unified_format.py | 2 - test/csot/deprecated-options.json | 67 ++++++++++++++--- test/csot/global-timeoutMS.json | 20 ++++- test/csot/override-operation-timeoutMS.json | 40 ++++++++-- test/csot/tailable-awaitData.json | 82 ++++++++++++++++++++- test/unified_format.py | 2 - 6 files changed, 183 insertions(+), 30 deletions(-) diff --git a/test/asynchronous/unified_format.py b/test/asynchronous/unified_format.py index b06654b328..5f01642a44 100644 --- a/test/asynchronous/unified_format.py +++ b/test/asynchronous/unified_format.py @@ -591,8 +591,6 @@ def maybe_skip_test(self, spec): self.skipTest("CSOT not implemented for watch()") if "cursors" in class_name: self.skipTest("CSOT not implemented for cursors") - if "dropindex on collection" in description: - self.skipTest("PYTHON-5491") if ( "tailable" in class_name or "tailable" in description diff --git a/test/csot/deprecated-options.json b/test/csot/deprecated-options.json index d3e4631ff4..647e1bf792 100644 --- a/test/csot/deprecated-options.json +++ b/test/csot/deprecated-options.json @@ -6750,16 +6750,23 @@ } } }, + { + "name": "createIndex", + "object": "collection", + "arguments": { + "keys": { + "x": 1 + }, + "timeoutMS": 100000, + "name": "x_1" + } + }, { "name": "dropIndex", "object": "collection", "arguments": { "timeoutMS": 100000, "name": "x_1" - }, - "expectError": { - "isClientError": false, - "isTimeoutError": false } } ] @@ -6815,16 +6822,23 @@ ] } }, + { + "name": "createIndex", + "object": "collection", + "arguments": { + "keys": { + "x": 1 + }, + "timeoutMS": 100000, + "name": "x_1" + } + }, { "name": "dropIndex", "object": "collection", "arguments": { "timeoutMS": 100000, "name": "x_1" - }, - "expectError": { - "isClientError": false, - "isTimeoutError": false } } ], @@ -6832,6 +6846,12 @@ { "client": "client", "events": [ + { + "commandStartedEvent": { + "commandName": "createIndexes", + "databaseName": "test" + } + }, { "commandStartedEvent": { "commandName": "dropIndexes", @@ -6903,6 +6923,16 @@ ] } }, + { + "name": "createIndex", + "object": "collection", + "arguments": { + "keys": { + "x": 1 + }, + "name": "x_1" + } + }, { "name": "dropIndex", "object": "collection", @@ -6910,10 +6940,6 @@ "timeoutMS": 1000, "maxTimeMS": 5000, "name": "x_1" - }, - "expectError": { - "isClientError": false, - "isTimeoutError": false } } ], @@ -6921,6 +6947,12 @@ { "client": "client", "events": [ + { + "commandStartedEvent": { + "commandName": "createIndexes", + "databaseName": "test" + } + }, { "commandStartedEvent": { "commandName": "dropIndexes", @@ -7003,6 +7035,17 @@ } } }, + { + "name": "createIndex", + "object": "collection", + "arguments": { + "keys": { + "x": 1 + }, + "name": "x_1", + "timeoutMS": 100000 + } + }, { "name": "dropIndexes", "object": "collection", diff --git a/test/csot/global-timeoutMS.json b/test/csot/global-timeoutMS.json index 740bbad2e2..f1edbe68e3 100644 --- a/test/csot/global-timeoutMS.json +++ b/test/csot/global-timeoutMS.json @@ -5621,15 +5621,21 @@ } } }, + { + "name": "createIndex", + "object": "collection", + "arguments": { + "keys": { + "x": 1 + }, + "name": "x_1" + } + }, { "name": "dropIndex", "object": "collection", "arguments": { "name": "x_1" - }, - "expectError": { - "isClientError": false, - "isTimeoutError": false } } ], @@ -5637,6 +5643,12 @@ { "client": "client", "events": [ + { + "commandStartedEvent": { + "commandName": "createIndexes", + "databaseName": "test" + } + }, { "commandStartedEvent": { "commandName": "dropIndexes", diff --git a/test/csot/override-operation-timeoutMS.json b/test/csot/override-operation-timeoutMS.json index 6fa0bd802a..f33f876137 100644 --- a/test/csot/override-operation-timeoutMS.json +++ b/test/csot/override-operation-timeoutMS.json @@ -3378,15 +3378,23 @@ } } }, + { + "name": "createIndex", + "object": "collection", + "arguments": { + "keys": { + "x": 1 + }, + "timeoutMS": 1000, + "name": "x_1" + } + }, { "name": "dropIndex", "object": "collection", "arguments": { "timeoutMS": 1000, "name": "x_1" - }, - "expectError": { - "isTimeoutError": false } } ], @@ -3394,6 +3402,12 @@ { "client": "client", "events": [ + { + "commandStartedEvent": { + "commandName": "createIndexes", + "databaseName": "test" + } + }, { "commandStartedEvent": { "commandName": "dropIndexes", @@ -3436,15 +3450,23 @@ } } }, + { + "name": "createIndex", + "object": "collection", + "arguments": { + "keys": { + "x": 1 + }, + "timeoutMS": 0, + "name": "x_1" + } + }, { "name": "dropIndex", "object": "collection", "arguments": { "timeoutMS": 0, "name": "x_1" - }, - "expectError": { - "isTimeoutError": false } } ], @@ -3452,6 +3474,12 @@ { "client": "client", "events": [ + { + "commandStartedEvent": { + "commandName": "createIndexes", + "databaseName": "test" + } + }, { "commandStartedEvent": { "commandName": "dropIndexes", diff --git a/test/csot/tailable-awaitData.json b/test/csot/tailable-awaitData.json index 81683d3993..80e95ca906 100644 --- a/test/csot/tailable-awaitData.json +++ b/test/csot/tailable-awaitData.json @@ -78,7 +78,7 @@ ] }, { - "description": "error if maxAwaitTimeMS is greater than timeoutMS", + "description": "error on find if maxAwaitTimeMS is greater than timeoutMS", "operations": [ { "name": "find", @@ -90,13 +90,50 @@ "maxAwaitTimeMS": 10 }, "expectError": { - "isClientError": true + "isClientError": true, + "isTimeoutError": false + } + } + ] + }, + { + "description": "error on aggregate if maxAwaitTimeMS is greater than timeoutMS", + "operations": [ + { + "name": "aggregate", + "object": "collection", + "arguments": { + "pipeline": [], + "timeoutMS": 5, + "maxAwaitTimeMS": 10 + }, + "expectError": { + "isClientError": true, + "isTimeoutError": false + } + } + ] + }, + { + "description": "error on watch if maxAwaitTimeMS is greater than timeoutMS", + "operations": [ + { + "name": "createChangeStream", + "object": "collection", + "arguments": { + "pipeline": [], + "timeoutMS": 5, + "maxAwaitTimeMS": 10 + }, + "expectError": { + "isClientError": true, + "isTimeoutError": false } } ] }, { - "description": "error if maxAwaitTimeMS is equal to timeoutMS", + "description": "error on find if maxAwaitTimeMS is equal to timeoutMS", "operations": [ { "name": "find", @@ -108,7 +145,44 @@ "maxAwaitTimeMS": 5 }, "expectError": { - "isClientError": true + "isClientError": true, + "isTimeoutError": false + } + } + ] + }, + { + "description": "error on aggregate if maxAwaitTimeMS is equal to timeoutMS", + "operations": [ + { + "name": "aggregate", + "object": "collection", + "arguments": { + "pipeline": [], + "timeoutMS": 5, + "maxAwaitTimeMS": 5 + }, + "expectError": { + "isClientError": true, + "isTimeoutError": false + } + } + ] + }, + { + "description": "error on watch if maxAwaitTimeMS is equal to timeoutMS", + "operations": [ + { + "name": "createChangeStream", + "object": "collection", + "arguments": { + "pipeline": [], + "timeoutMS": 5, + "maxAwaitTimeMS": 5 + }, + "expectError": { + "isClientError": true, + "isTimeoutError": false } } ] diff --git a/test/unified_format.py b/test/unified_format.py index 2cbc581aca..375d845783 100644 --- a/test/unified_format.py +++ b/test/unified_format.py @@ -590,8 +590,6 @@ def maybe_skip_test(self, spec): self.skipTest("CSOT not implemented for watch()") if "cursors" in class_name: self.skipTest("CSOT not implemented for cursors") - if "dropindex on collection" in description: - self.skipTest("PYTHON-5491") if ( "tailable" in class_name or "tailable" in description From 5e96353797f081ad2090270818a8808d37ebac40 Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Thu, 21 Aug 2025 06:51:00 -0700 Subject: [PATCH 06/53] PYTHON-5508 - Add built-in DecimalEncoder and DecimalDecoder (#2499) --- bson/decimal128.py | 39 ++++++++++++++++++++++++++ doc/changelog.rst | 7 +++++ test/asynchronous/test_custom_types.py | 25 ++--------------- test/test_custom_types.py | 25 ++--------------- 4 files changed, 50 insertions(+), 46 deletions(-) diff --git a/bson/decimal128.py b/bson/decimal128.py index 92c054d878..7480f94d0a 100644 --- a/bson/decimal128.py +++ b/bson/decimal128.py @@ -20,8 +20,11 @@ import decimal import struct +from decimal import Decimal from typing import Any, Sequence, Tuple, Type, Union +from bson.codec_options import TypeDecoder, TypeEncoder + _PACK_64 = struct.Struct(" Type[Decimal]: + return Decimal + + def transform_python(self, value: Any) -> Decimal128: + return Decimal128(value) + + +class DecimalDecoder(TypeDecoder): + """Converts BSON :class:`Decimal128` to Python :class:`decimal.Decimal`. + + For example:: + opts = CodecOptions(type_registry=TypeRegistry([DecimalDecoder()])) + bson.decode(data, codec_options=opts) + + .. versionadded:: 4.15 + """ + + @property + def bson_type(self) -> Type[Decimal128]: + return Decimal128 + + def transform_bson(self, value: Any) -> decimal.Decimal: + return value.to_decimal() + + def create_decimal128_context() -> decimal.Context: """Returns an instance of :class:`decimal.Context` appropriate for working with IEEE-754 128-bit decimal floating point values. diff --git a/doc/changelog.rst b/doc/changelog.rst index e41ecc7e1b..305c989106 100644 --- a/doc/changelog.rst +++ b/doc/changelog.rst @@ -1,5 +1,12 @@ Changelog ========= +Changes in Version 4.15.0 (XXXX/XX/XX) +-------------------------------------- +PyMongo 4.15 brings a number of changes including: + +- Added :class:`bson.decimal128.DecimalEncoder` and :class:`bson.decimal128.DecimalDecoder` + to support encoding and decoding of BSON Decimal128 values to decimal.Decimal values using the TypeRegistry API. + Changes in Version 4.14.1 (2025/08/19) -------------------------------------- diff --git a/test/asynchronous/test_custom_types.py b/test/asynchronous/test_custom_types.py index 385f755a1d..82c54512cc 100644 --- a/test/asynchronous/test_custom_types.py +++ b/test/asynchronous/test_custom_types.py @@ -23,6 +23,7 @@ from random import random from typing import Any, Tuple, Type, no_type_check +from bson.decimal128 import DecimalDecoder, DecimalEncoder from gridfs.asynchronous.grid_file import AsyncGridIn, AsyncGridOut sys.path[0:0] = [""] @@ -59,29 +60,7 @@ _IS_SYNC = False -class DecimalEncoder(TypeEncoder): - @property - def python_type(self): - return Decimal - - def transform_python(self, value): - return Decimal128(value) - - -class DecimalDecoder(TypeDecoder): - @property - def bson_type(self): - return Decimal128 - - def transform_bson(self, value): - return value.to_decimal() - - -class DecimalCodec(DecimalDecoder, DecimalEncoder): - pass - - -DECIMAL_CODECOPTS = CodecOptions(type_registry=TypeRegistry([DecimalCodec()])) +DECIMAL_CODECOPTS = CodecOptions(type_registry=TypeRegistry([DecimalEncoder(), DecimalDecoder()])) class UndecipherableInt64Type: diff --git a/test/test_custom_types.py b/test/test_custom_types.py index 7360f2b18b..aba6b55119 100644 --- a/test/test_custom_types.py +++ b/test/test_custom_types.py @@ -23,6 +23,7 @@ from random import random from typing import Any, Tuple, Type, no_type_check +from bson.decimal128 import DecimalDecoder, DecimalEncoder from gridfs.synchronous.grid_file import GridIn, GridOut sys.path[0:0] = [""] @@ -59,29 +60,7 @@ _IS_SYNC = True -class DecimalEncoder(TypeEncoder): - @property - def python_type(self): - return Decimal - - def transform_python(self, value): - return Decimal128(value) - - -class DecimalDecoder(TypeDecoder): - @property - def bson_type(self): - return Decimal128 - - def transform_bson(self, value): - return value.to_decimal() - - -class DecimalCodec(DecimalDecoder, DecimalEncoder): - pass - - -DECIMAL_CODECOPTS = CodecOptions(type_registry=TypeRegistry([DecimalCodec()])) +DECIMAL_CODECOPTS = CodecOptions(type_registry=TypeRegistry([DecimalEncoder(), DecimalDecoder()])) class UndecipherableInt64Type: From e08284bdca4e35121e618443741aae1d25153fc8 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 21 Aug 2025 10:55:48 -0500 Subject: [PATCH 07/53] PYTHON-5456 Support text indexes with auto encryption (#2500) --- test/asynchronous/unified_format.py | 12 +- ...-Text-cleanupStructuredEncryptionData.json | 219 +++++++ ...-Text-compactStructuredEncryptionData.json | 261 +++++++++ .../spec/unified/QE-Text-prefixPreview.json | 338 +++++++++++ .../unified/QE-Text-substringPreview.json | 551 ++++++++++++++++++ .../spec/unified/QE-Text-suffixPreview.json | 338 +++++++++++ test/unified_format.py | 12 +- 7 files changed, 1727 insertions(+), 4 deletions(-) create mode 100644 test/client-side-encryption/spec/unified/QE-Text-cleanupStructuredEncryptionData.json create mode 100644 test/client-side-encryption/spec/unified/QE-Text-compactStructuredEncryptionData.json create mode 100644 test/client-side-encryption/spec/unified/QE-Text-prefixPreview.json create mode 100644 test/client-side-encryption/spec/unified/QE-Text-substringPreview.json create mode 100644 test/client-side-encryption/spec/unified/QE-Text-suffixPreview.json diff --git a/test/asynchronous/unified_format.py b/test/asynchronous/unified_format.py index 5f01642a44..9bd0fabdb8 100644 --- a/test/asynchronous/unified_format.py +++ b/test/asynchronous/unified_format.py @@ -159,6 +159,14 @@ async def is_run_on_requirement_satisfied(requirement): if req_csfle is True: min_version_satisfied = Version.from_string("4.2") <= server_version csfle_satisfied = _HAVE_PYMONGOCRYPT and min_version_satisfied + elif isinstance(req_csfle, dict) and "minLibmongocryptVersion" in req_csfle: + csfle_satisfied = False + req_version = req_csfle["minLibmongocryptVersion"] + if _HAVE_PYMONGOCRYPT: + from pymongocrypt import libmongocrypt_version + + if Version.from_string(libmongocrypt_version()) >= Version.from_string(req_version): + csfle_satisfied = True return ( topology_satisfied @@ -450,7 +458,7 @@ class UnifiedSpecTestMixinV1(AsyncIntegrationTest): a class attribute ``TEST_SPEC``. """ - SCHEMA_VERSION = Version.from_string("1.23") + SCHEMA_VERSION = Version.from_string("1.25") RUN_ON_LOAD_BALANCER = True TEST_SPEC: Any TEST_PATH = "" # This gets filled in by generate_test_classes @@ -1545,7 +1553,7 @@ class SpecTestBase(with_metaclass(UnifiedSpecTestMeta)): # type: ignore # Add "encryption" marker if the "csfle" runOnRequirement is set. for req in test_spec.get("runOnRequirements", []): - if req.get("csfle", False): + if "csfle" in req: base = pytest.mark.encryption(base) return base diff --git a/test/client-side-encryption/spec/unified/QE-Text-cleanupStructuredEncryptionData.json b/test/client-side-encryption/spec/unified/QE-Text-cleanupStructuredEncryptionData.json new file mode 100644 index 0000000000..24f33ab3ec --- /dev/null +++ b/test/client-side-encryption/spec/unified/QE-Text-cleanupStructuredEncryptionData.json @@ -0,0 +1,219 @@ +{ + "description": "QE-Text-cleanupStructuredEncryptionData", + "schemaVersion": "1.25", + "runOnRequirements": [ + { + "minServerVersion": "8.2.0", + "topologies": [ + "replicaset", + "sharded", + "load-balanced" + ], + "csfle": { + "minLibmongocryptVersion": "1.15.0" + } + } + ], + "createEntities": [ + { + "client": { + "id": "client", + "autoEncryptOpts": { + "keyVaultNamespace": "keyvault.datakeys", + "kmsProviders": { + "local": { + "key": "Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk" + } + } + }, + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "database": { + "id": "db", + "client": "client", + "databaseName": "db" + } + }, + { + "collection": { + "id": "coll", + "database": "db", + "collectionName": "coll" + } + } + ], + "initialData": [ + { + "databaseName": "keyvault", + "collectionName": "datakeys", + "documents": [ + { + "_id": { + "$binary": { + "base64": "q83vqxI0mHYSNBI0VniQEg==", + "subType": "04" + } + }, + "keyMaterial": { + "$binary": { + "base64": "HBk9BWihXExNDvTp1lUxOuxuZK2Pe2ZdVdlsxPEBkiO1bS4mG5NNDsQ7zVxJAH8BtdOYp72Ku4Y3nwc0BUpIKsvAKX4eYXtlhv5zUQxWdeNFhg9qK7qb8nqhnnLeT0f25jFSqzWJoT379hfwDeu0bebJHr35QrJ8myZdPMTEDYF08QYQ48ShRBli0S+QzBHHAQiM2iJNr4svg2WR8JSeWQ==", + "subType": "00" + } + }, + "creationDate": { + "$date": { + "$numberLong": "1648914851981" + } + }, + "updateDate": { + "$date": { + "$numberLong": "1648914851981" + } + }, + "status": { + "$numberInt": "0" + }, + "masterKey": { + "provider": "local" + } + } + ] + }, + { + "databaseName": "db", + "collectionName": "coll", + "documents": [], + "createOptions": { + "encryptedFields": { + "fields": [ + { + "keyId": { + "$binary": { + "base64": "q83vqxI0mHYSNBI0VniQEg==", + "subType": "04" + } + }, + "path": "encryptedText", + "bsonType": "string", + "queries": [ + { + "queryType": "suffixPreview", + "contention": { + "$numberLong": "0" + }, + "strMinQueryLength": { + "$numberLong": "3" + }, + "strMaxQueryLength": { + "$numberLong": "30" + }, + "caseSensitive": true, + "diacriticSensitive": true + } + ] + } + ] + } + } + } + ], + "tests": [ + { + "description": "QE Text cleanupStructuredEncryptionData works", + "operations": [ + { + "name": "runCommand", + "object": "db", + "arguments": { + "command": { + "cleanupStructuredEncryptionData": "coll" + }, + "commandName": "cleanupStructuredEncryptionData" + }, + "expectResult": { + "ok": 1 + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "command": { + "listCollections": 1, + "filter": { + "name": "coll" + } + }, + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "command": { + "find": "datakeys", + "filter": { + "$or": [ + { + "_id": { + "$in": [ + { + "$binary": { + "base64": "q83vqxI0mHYSNBI0VniQEg==", + "subType": "04" + } + } + ] + } + }, + { + "keyAltNames": { + "$in": [] + } + } + ] + }, + "$db": "keyvault", + "readConcern": { + "level": "majority" + } + }, + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "command": { + "cleanupStructuredEncryptionData": "coll", + "cleanupTokens": { + "encryptedText": { + "ecoc": { + "$binary": { + "base64": "SWO8WEoZ2r2Kx/muQKb7+COizy85nIIUFiHh4K9kcvA=", + "subType": "00" + } + }, + "anchorPaddingToken": { + "$binary": { + "base64": "YAiF7Iwhqq1UyfxPvm70xfQJtrIRPrjfD2yRLG1+saQ=", + "subType": "00" + } + } + } + } + }, + "commandName": "cleanupStructuredEncryptionData" + } + } + ] + } + ] + } + ] +} diff --git a/test/client-side-encryption/spec/unified/QE-Text-compactStructuredEncryptionData.json b/test/client-side-encryption/spec/unified/QE-Text-compactStructuredEncryptionData.json new file mode 100644 index 0000000000..c7abfe2d4b --- /dev/null +++ b/test/client-side-encryption/spec/unified/QE-Text-compactStructuredEncryptionData.json @@ -0,0 +1,261 @@ +{ + "description": "QE-Text-compactStructuredEncryptionData", + "schemaVersion": "1.25", + "runOnRequirements": [ + { + "minServerVersion": "8.2.0", + "topologies": [ + "replicaset", + "sharded", + "load-balanced" + ], + "csfle": { + "minLibmongocryptVersion": "1.15.0" + } + } + ], + "createEntities": [ + { + "client": { + "id": "client", + "autoEncryptOpts": { + "keyVaultNamespace": "keyvault.datakeys", + "kmsProviders": { + "local": { + "key": "Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk" + } + } + }, + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "database": { + "id": "db", + "client": "client", + "databaseName": "db" + } + }, + { + "collection": { + "id": "coll", + "database": "db", + "collectionName": "coll" + } + } + ], + "initialData": [ + { + "databaseName": "keyvault", + "collectionName": "datakeys", + "documents": [ + { + "_id": { + "$binary": { + "base64": "q83vqxI0mHYSNBI0VniQEg==", + "subType": "04" + } + }, + "keyMaterial": { + "$binary": { + "base64": "HBk9BWihXExNDvTp1lUxOuxuZK2Pe2ZdVdlsxPEBkiO1bS4mG5NNDsQ7zVxJAH8BtdOYp72Ku4Y3nwc0BUpIKsvAKX4eYXtlhv5zUQxWdeNFhg9qK7qb8nqhnnLeT0f25jFSqzWJoT379hfwDeu0bebJHr35QrJ8myZdPMTEDYF08QYQ48ShRBli0S+QzBHHAQiM2iJNr4svg2WR8JSeWQ==", + "subType": "00" + } + }, + "creationDate": { + "$date": { + "$numberLong": "1648914851981" + } + }, + "updateDate": { + "$date": { + "$numberLong": "1648914851981" + } + }, + "status": { + "$numberInt": "0" + }, + "masterKey": { + "provider": "local" + } + } + ] + }, + { + "databaseName": "db", + "collectionName": "coll", + "documents": [], + "createOptions": { + "encryptedFields": { + "fields": [ + { + "keyId": { + "$binary": { + "base64": "q83vqxI0mHYSNBI0VniQEg==", + "subType": "04" + } + }, + "path": "encryptedText", + "bsonType": "string", + "queries": [ + { + "queryType": "suffixPreview", + "contention": { + "$numberLong": "0" + }, + "strMinQueryLength": { + "$numberLong": "3" + }, + "strMaxQueryLength": { + "$numberLong": "30" + }, + "caseSensitive": true, + "diacriticSensitive": true + } + ] + } + ] + } + } + } + ], + "tests": [ + { + "description": "QE Text compactStructuredEncryptionData works", + "operations": [ + { + "name": "runCommand", + "object": "db", + "arguments": { + "command": { + "compactStructuredEncryptionData": "coll" + }, + "commandName": "compactStructuredEncryptionData" + }, + "expectResult": { + "ok": 1 + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "command": { + "listCollections": 1, + "filter": { + "name": "coll" + } + }, + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "command": { + "find": "datakeys", + "filter": { + "$or": [ + { + "_id": { + "$in": [ + { + "$binary": { + "base64": "q83vqxI0mHYSNBI0VniQEg==", + "subType": "04" + } + } + ] + } + }, + { + "keyAltNames": { + "$in": [] + } + } + ] + }, + "$db": "keyvault", + "readConcern": { + "level": "majority" + } + }, + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "command": { + "compactStructuredEncryptionData": "coll", + "encryptionInformation": { + "type": { + "$numberInt": "1" + }, + "schema": { + "db.coll": { + "fields": [ + { + "keyId": { + "$binary": { + "base64": "q83vqxI0mHYSNBI0VniQEg==", + "subType": "04" + } + }, + "path": "encryptedText", + "bsonType": "string", + "queries": [ + { + "queryType": "suffixPreview", + "contention": { + "$numberLong": "0" + }, + "strMinQueryLength": { + "$numberLong": "3" + }, + "strMaxQueryLength": { + "$numberLong": "30" + }, + "caseSensitive": true, + "diacriticSensitive": true + } + ] + } + ], + "strEncodeVersion": { + "$numberInt": "1" + }, + "escCollection": "enxcol_.coll.esc", + "ecocCollection": "enxcol_.coll.ecoc" + } + } + }, + "compactionTokens": { + "encryptedText": { + "ecoc": { + "$binary": { + "base64": "SWO8WEoZ2r2Kx/muQKb7+COizy85nIIUFiHh4K9kcvA=", + "subType": "00" + } + }, + "anchorPaddingToken": { + "$binary": { + "base64": "YAiF7Iwhqq1UyfxPvm70xfQJtrIRPrjfD2yRLG1+saQ=", + "subType": "00" + } + } + } + } + }, + "commandName": "compactStructuredEncryptionData" + } + } + ] + } + ] + } + ] +} diff --git a/test/client-side-encryption/spec/unified/QE-Text-prefixPreview.json b/test/client-side-encryption/spec/unified/QE-Text-prefixPreview.json new file mode 100644 index 0000000000..7279385743 --- /dev/null +++ b/test/client-side-encryption/spec/unified/QE-Text-prefixPreview.json @@ -0,0 +1,338 @@ +{ + "description": "QE-Text-prefixPreview", + "schemaVersion": "1.25", + "runOnRequirements": [ + { + "minServerVersion": "8.2.0", + "topologies": [ + "replicaset", + "sharded", + "load-balanced" + ], + "csfle": { + "minLibmongocryptVersion": "1.15.0" + } + } + ], + "createEntities": [ + { + "client": { + "id": "client", + "autoEncryptOpts": { + "keyVaultNamespace": "keyvault.datakeys", + "kmsProviders": { + "local": { + "key": "Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk" + } + } + }, + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "database": { + "id": "db", + "client": "client", + "databaseName": "db" + } + }, + { + "collection": { + "id": "coll", + "database": "db", + "collectionName": "coll" + } + } + ], + "initialData": [ + { + "databaseName": "keyvault", + "collectionName": "datakeys", + "documents": [ + { + "_id": { + "$binary": { + "base64": "q83vqxI0mHYSNBI0VniQEg==", + "subType": "04" + } + }, + "keyMaterial": { + "$binary": { + "base64": "HBk9BWihXExNDvTp1lUxOuxuZK2Pe2ZdVdlsxPEBkiO1bS4mG5NNDsQ7zVxJAH8BtdOYp72Ku4Y3nwc0BUpIKsvAKX4eYXtlhv5zUQxWdeNFhg9qK7qb8nqhnnLeT0f25jFSqzWJoT379hfwDeu0bebJHr35QrJ8myZdPMTEDYF08QYQ48ShRBli0S+QzBHHAQiM2iJNr4svg2WR8JSeWQ==", + "subType": "00" + } + }, + "creationDate": { + "$date": { + "$numberLong": "1648914851981" + } + }, + "updateDate": { + "$date": { + "$numberLong": "1648914851981" + } + }, + "status": { + "$numberInt": "0" + }, + "masterKey": { + "provider": "local" + } + } + ] + }, + { + "databaseName": "db", + "collectionName": "coll", + "documents": [], + "createOptions": { + "encryptedFields": { + "fields": [ + { + "keyId": { + "$binary": { + "base64": "q83vqxI0mHYSNBI0VniQEg==", + "subType": "04" + } + }, + "path": "encryptedText", + "bsonType": "string", + "queries": [ + { + "queryType": "prefixPreview", + "contention": { + "$numberLong": "0" + }, + "strMinQueryLength": { + "$numberLong": "3" + }, + "strMaxQueryLength": { + "$numberLong": "30" + }, + "caseSensitive": true, + "diacriticSensitive": true + } + ] + } + ] + } + } + } + ], + "tests": [ + { + "description": "Insert QE prefixPreview", + "operations": [ + { + "name": "insertOne", + "arguments": { + "document": { + "_id": 1, + "encryptedText": "foobar" + } + }, + "object": "coll" + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "command": { + "listCollections": 1, + "filter": { + "name": "coll" + } + }, + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "command": { + "find": "datakeys", + "filter": { + "$or": [ + { + "_id": { + "$in": [ + { + "$binary": { + "base64": "q83vqxI0mHYSNBI0VniQEg==", + "subType": "04" + } + } + ] + } + }, + { + "keyAltNames": { + "$in": [] + } + } + ] + }, + "$db": "keyvault", + "readConcern": { + "level": "majority" + } + }, + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "command": { + "insert": "coll", + "documents": [ + { + "_id": 1, + "encryptedText": { + "$$type": "binData" + } + } + ], + "ordered": true + }, + "commandName": "insert" + } + } + ] + } + ] + }, + { + "description": "Query with matching $encStrStartsWith", + "operations": [ + { + "name": "insertOne", + "arguments": { + "document": { + "_id": 1, + "encryptedText": "foobar" + } + }, + "object": "coll" + }, + { + "name": "find", + "arguments": { + "filter": { + "$expr": { + "$encStrStartsWith": { + "input": "$encryptedText", + "prefix": "foo" + } + } + } + }, + "object": "coll", + "expectResult": [ + { + "_id": { + "$numberInt": "1" + }, + "encryptedText": "foobar", + "__safeContent__": [ + { + "$binary": { + "base64": "wpaMBVDjL4bHf9EtSP52PJFzyNn1R19+iNI/hWtvzdk=", + "subType": "00" + } + }, + { + "$binary": { + "base64": "fmUMXTMV/XRiN0IL3VXxSEn6SQG9E6Po30kJKB8JJlQ=", + "subType": "00" + } + }, + { + "$binary": { + "base64": "vZIDMiFDgjmLNYVrrbnq1zT4hg7sGpe/PMtighSsnRc=", + "subType": "00" + } + }, + { + "$binary": { + "base64": "26Z5G+sHTzV3D7F8Y0m08389USZ2afinyFV3ez9UEBQ=", + "subType": "00" + } + }, + { + "$binary": { + "base64": "q/JEq8of7bE0QE5Id0XuOsNQ4qVpANYymcPQDUL2Ywk=", + "subType": "00" + } + }, + { + "$binary": { + "base64": "Uvvv46LkfbgLoPqZ6xTBzpgoYRTM6FUgRdqZ9eaVojI=", + "subType": "00" + } + }, + { + "$binary": { + "base64": "nMxdq2lladuBJA3lv3JC2MumIUtRJBNJVLp3PVE6nQk=", + "subType": "00" + } + }, + { + "$binary": { + "base64": "hS3V0qq5CF/SkTl3ZWWWgXcAJ8G5yGtkY2RwcHNc5Oc=", + "subType": "00" + } + }, + { + "$binary": { + "base64": "McgwYUxfKj5+4D0vskZymy4KA82s71MR25iV/Enutww=", + "subType": "00" + } + }, + { + "$binary": { + "base64": "Ciqdk1b+t+Vrr6oIlFFk0Zdym5BPmwN3glQ0/VcsVdM=", + "subType": "00" + } + } + ] + } + ] + } + ] + }, + { + "description": "Query with non-matching $encStrStartsWith", + "operations": [ + { + "name": "insertOne", + "arguments": { + "document": { + "_id": 1, + "encryptedText": "foobar" + } + }, + "object": "coll" + }, + { + "name": "find", + "arguments": { + "filter": { + "$expr": { + "$encStrStartsWith": { + "input": "$encryptedText", + "prefix": "bar" + } + } + } + }, + "object": "coll", + "expectResult": [] + } + ] + } + ] +} diff --git a/test/client-side-encryption/spec/unified/QE-Text-substringPreview.json b/test/client-side-encryption/spec/unified/QE-Text-substringPreview.json new file mode 100644 index 0000000000..6a8f133eac --- /dev/null +++ b/test/client-side-encryption/spec/unified/QE-Text-substringPreview.json @@ -0,0 +1,551 @@ +{ + "description": "QE-Text-substringPreview", + "schemaVersion": "1.25", + "runOnRequirements": [ + { + "minServerVersion": "8.2.0", + "topologies": [ + "replicaset", + "sharded", + "load-balanced" + ], + "csfle": { + "minLibmongocryptVersion": "1.15.0" + } + } + ], + "createEntities": [ + { + "client": { + "id": "client", + "autoEncryptOpts": { + "keyVaultNamespace": "keyvault.datakeys", + "kmsProviders": { + "local": { + "key": "Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk" + } + } + }, + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "database": { + "id": "db", + "client": "client", + "databaseName": "db" + } + }, + { + "collection": { + "id": "coll", + "database": "db", + "collectionName": "coll" + } + } + ], + "initialData": [ + { + "databaseName": "keyvault", + "collectionName": "datakeys", + "documents": [ + { + "_id": { + "$binary": { + "base64": "q83vqxI0mHYSNBI0VniQEg==", + "subType": "04" + } + }, + "keyMaterial": { + "$binary": { + "base64": "HBk9BWihXExNDvTp1lUxOuxuZK2Pe2ZdVdlsxPEBkiO1bS4mG5NNDsQ7zVxJAH8BtdOYp72Ku4Y3nwc0BUpIKsvAKX4eYXtlhv5zUQxWdeNFhg9qK7qb8nqhnnLeT0f25jFSqzWJoT379hfwDeu0bebJHr35QrJ8myZdPMTEDYF08QYQ48ShRBli0S+QzBHHAQiM2iJNr4svg2WR8JSeWQ==", + "subType": "00" + } + }, + "creationDate": { + "$date": { + "$numberLong": "1648914851981" + } + }, + "updateDate": { + "$date": { + "$numberLong": "1648914851981" + } + }, + "status": { + "$numberInt": "0" + }, + "masterKey": { + "provider": "local" + } + } + ] + }, + { + "databaseName": "db", + "collectionName": "coll", + "documents": [], + "createOptions": { + "encryptedFields": { + "fields": [ + { + "keyId": { + "$binary": { + "base64": "q83vqxI0mHYSNBI0VniQEg==", + "subType": "04" + } + }, + "path": "encryptedText", + "bsonType": "string", + "queries": [ + { + "queryType": "substringPreview", + "contention": { + "$numberLong": "0" + }, + "strMinQueryLength": { + "$numberLong": "3" + }, + "strMaxQueryLength": { + "$numberLong": "10" + }, + "strMaxLength": { + "$numberLong": "20" + }, + "caseSensitive": true, + "diacriticSensitive": true + } + ] + } + ] + } + } + } + ], + "tests": [ + { + "description": "Insert QE suffixPreview", + "operations": [ + { + "name": "insertOne", + "arguments": { + "document": { + "_id": 1, + "encryptedText": "foobar" + } + }, + "object": "coll" + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "command": { + "listCollections": 1, + "filter": { + "name": "coll" + } + }, + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "command": { + "find": "datakeys", + "filter": { + "$or": [ + { + "_id": { + "$in": [ + { + "$binary": { + "base64": "q83vqxI0mHYSNBI0VniQEg==", + "subType": "04" + } + } + ] + } + }, + { + "keyAltNames": { + "$in": [] + } + } + ] + }, + "$db": "keyvault", + "readConcern": { + "level": "majority" + } + }, + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "command": { + "insert": "coll", + "documents": [ + { + "_id": 1, + "encryptedText": { + "$$type": "binData" + } + } + ], + "ordered": true + }, + "commandName": "insert" + } + } + ] + } + ] + }, + { + "description": "Query with matching $encStrContains", + "operations": [ + { + "name": "insertOne", + "arguments": { + "document": { + "_id": 1, + "encryptedText": "foobar" + } + }, + "object": "coll" + }, + { + "name": "find", + "arguments": { + "filter": { + "$expr": { + "$encStrContains": { + "input": "$encryptedText", + "substring": "oba" + } + } + } + }, + "object": "coll", + "expectResult": [ + { + "_id": { + "$numberInt": "1" + }, + "encryptedText": "foobar", + "__safeContent__": [ + { + "$binary": { + "base64": "wpaMBVDjL4bHf9EtSP52PJFzyNn1R19+iNI/hWtvzdk=", + "subType": "00" + } + }, + { + "$binary": { + "base64": "IpY3x/jjm8j/74jAdUhgxdM5hk68zR0zv/lTKm/72Vg=", + "subType": "00" + } + }, + { + "$binary": { + "base64": "G+ky260C6QiOfIxKz14FmaMbAxvui1BKJO/TnLOHlGk=", + "subType": "00" + } + }, + { + "$binary": { + "base64": "7dv3gAKe9vwJMZmpB40pRCwRTmc7ds9UkGhxH8j084E=", + "subType": "00" + } + }, + { + "$binary": { + "base64": "o0V+Efn6x8XQdE80F1tztNaT3qxHjcsd9DOQ47BtmQk=", + "subType": "00" + } + }, + { + "$binary": { + "base64": "sJvrCjyVot7PIZFsdRehWFANKAj6fmBaj3FLbz/dZLE=", + "subType": "00" + } + }, + { + "$binary": { + "base64": "e98auxFmu02h5MfBIARk29MI7hSmvN3F9DaQ0xjqoEM=", + "subType": "00" + } + }, + { + "$binary": { + "base64": "US83krGNov/ezL6IhsY5eEOCxv1xUPDIEL/nmY0IKi0=", + "subType": "00" + } + }, + { + "$binary": { + "base64": "P2Aq5+OHZPG0CWIdmZvWq9c/18ZKVYW3vbxd+WU/TXU=", + "subType": "00" + } + }, + { + "$binary": { + "base64": "8AdPRPnSzcd5uhq4TZfNvNeF0XjLNVwAsJJMTtktw84=", + "subType": "00" + } + }, + { + "$binary": { + "base64": "9O6u/G51I4ZHFLhL4ZLuudbr0s202A2QnPfThmOXPhI=", + "subType": "00" + } + }, + { + "$binary": { + "base64": "N7AjYVyVlv6+lVSTM+cIxRL3SMgs3G5LgxSs+jrgDkI=", + "subType": "00" + } + }, + { + "$binary": { + "base64": "RbGF7dQbPGYQFd9DDO1hPz1UlLOJ77FAC6NsjGwJeos=", + "subType": "00" + } + }, + { + "$binary": { + "base64": "m7srHMgKm6kZwsNx8rc45pmw0/9Qro6xuQ8lZS3+RYk=", + "subType": "00" + } + }, + { + "$binary": { + "base64": "K75CNU3JyKFqZWPiIsVi4+n7DhYmcPl/nEhQ3d88mVI=", + "subType": "00" + } + }, + { + "$binary": { + "base64": "c7bwGpUZc/7JzEnMS7qQ/TPuXZyrmMihFaAV6zIqbZc=", + "subType": "00" + } + }, + { + "$binary": { + "base64": "rDvEdUgEk8u4Srt3ETokWs2FXcnyJaRGQ+NbkFwi2rQ=", + "subType": "00" + } + }, + { + "$binary": { + "base64": "VcdZj9zfveRBRlpCR2OYWau2+GokOFb73TE3gpElNiU=", + "subType": "00" + } + }, + { + "$binary": { + "base64": "eOa9o2xfA6OgkbYUxd6wQJicaeN6guhy2V66W3ALsaA=", + "subType": "00" + } + }, + { + "$binary": { + "base64": "1xGkJh+um70XiRd8lKLDtyHgDqrf7/59Mg7X0+KZh8k=", + "subType": "00" + } + }, + { + "$binary": { + "base64": "OSvllqHxycbcZN4phR6NDujY3ttA59o7nQJ6V9eJpX0=", + "subType": "00" + } + }, + { + "$binary": { + "base64": "ZTX1pyk8Vdw0BSbJx7GeJNcQf3tGKxbrrNSTqBqUWkg=", + "subType": "00" + } + }, + { + "$binary": { + "base64": "cn7V05zb5iXwYrePGMHztC+GRq+Tj8IMpRDraauPhSE=", + "subType": "00" + } + }, + { + "$binary": { + "base64": "E9bV9KyrZxHJSUmMg0HrDK4gGN+75ruelAnrM6hXQgY=", + "subType": "00" + } + }, + { + "$binary": { + "base64": "WrssTNmdgXoTGpbaF0JLRCGH6cDQuz1XEFNTy98nrb0=", + "subType": "00" + } + }, + { + "$binary": { + "base64": "jZmyOJP35dsxQ/OY5U4ISpVRIYr8iedNfcwZiKt29Qc=", + "subType": "00" + } + }, + { + "$binary": { + "base64": "d2mocORMbX9MX+/itAW8r1kxVw2/uii4vzXtc+2CIRQ=", + "subType": "00" + } + }, + { + "$binary": { + "base64": "JBnJy58eRPhDo3DuZvsHbvQDiHXxdtAx1Eif66k5SfA=", + "subType": "00" + } + }, + { + "$binary": { + "base64": "OjbDulC8s62v0pgweBSsQqtJjJBwH5JinfJpj7nVr+A=", + "subType": "00" + } + }, + { + "$binary": { + "base64": "85i7KT2GP9nSda3Gsil5LKubhq0LDtc22pxBxHpR+nE=", + "subType": "00" + } + }, + { + "$binary": { + "base64": "u9Fvsclwrs9lwIcMPV/fMZD7L3d5anSfJQVjQb9mgLg=", + "subType": "00" + } + }, + { + "$binary": { + "base64": "LZ32ttmLJGOIw9oFaUCn3Sx5uHPTYJPSFpeGRWNqlUc=", + "subType": "00" + } + }, + { + "$binary": { + "base64": "mMsZvGEePTqtl0FJAL/jAdyWNQIlpwN61YIlZsSIZ6s=", + "subType": "00" + } + }, + { + "$binary": { + "base64": "XZcu1a/ZGsIzAl3j4MXQlLo4v2p7kvIqRHtIQYFmL6k=", + "subType": "00" + } + }, + { + "$binary": { + "base64": "Zse27LinlYCEnX6iTmJceI33mEJxFb0LdPxp0RiMOaQ=", + "subType": "00" + } + }, + { + "$binary": { + "base64": "vOv2Hgb2/sBpnX9XwFbIN6yDxhjchwlmczUf82W2tp4=", + "subType": "00" + } + }, + { + "$binary": { + "base64": "oQxZ9A6j3x5j6x1Jqw/N9tpP4rfWMjcV3y+a3PkrL7c=", + "subType": "00" + } + }, + { + "$binary": { + "base64": "/D7ew3EijyUnmT22awVFspcuyo3JChJcDeCPwpljzVM=", + "subType": "00" + } + }, + { + "$binary": { + "base64": "BEmmwqyamt9X3bcWDld61P01zquy8fBHAXq3SHAPP0M=", + "subType": "00" + } + }, + { + "$binary": { + "base64": "wygD9/kAo1KsRvtr1v+9/lvqoWdKwgh6gDHvAQfXPPk=", + "subType": "00" + } + }, + { + "$binary": { + "base64": "pRTKgF/uksrF1c1AcfSTY6ZhqBKVud1vIztQ4/36SLs=", + "subType": "00" + } + }, + { + "$binary": { + "base64": "C4iUo8oNJsjJ37BqnBgIgSQpf99X2Bb4W5MZEAmakHU=", + "subType": "00" + } + }, + { + "$binary": { + "base64": "icoE53jIq6Fu/YGKUiSUTYyZ8xdiTQY9jJiGxVJObpw=", + "subType": "00" + } + }, + { + "$binary": { + "base64": "oubCwk0V6G2RFWtcOnYDU4uUBoXBrhBRi4nZgrYj9JY=", + "subType": "00" + } + }, + { + "$binary": { + "base64": "IyqhQ9nGhzEi5YW2W6v1kGU5DY2u2qSqbM/qXdLdWVU=", + "subType": "00" + } + } + ] + } + ] + } + ] + }, + { + "description": "Query with non-matching $encStrContains", + "operations": [ + { + "name": "insertOne", + "arguments": { + "document": { + "_id": 1, + "encryptedText": "foobar" + } + }, + "object": "coll" + }, + { + "name": "find", + "arguments": { + "filter": { + "$expr": { + "$encStrContains": { + "input": "$encryptedText", + "substring": "blah" + } + } + } + }, + "object": "coll", + "expectResult": [] + } + ] + } + ] +} diff --git a/test/client-side-encryption/spec/unified/QE-Text-suffixPreview.json b/test/client-side-encryption/spec/unified/QE-Text-suffixPreview.json new file mode 100644 index 0000000000..deec5e63b0 --- /dev/null +++ b/test/client-side-encryption/spec/unified/QE-Text-suffixPreview.json @@ -0,0 +1,338 @@ +{ + "description": "QE-Text-suffixPreview", + "schemaVersion": "1.25", + "runOnRequirements": [ + { + "minServerVersion": "8.2.0", + "topologies": [ + "replicaset", + "sharded", + "load-balanced" + ], + "csfle": { + "minLibmongocryptVersion": "1.15.0" + } + } + ], + "createEntities": [ + { + "client": { + "id": "client", + "autoEncryptOpts": { + "keyVaultNamespace": "keyvault.datakeys", + "kmsProviders": { + "local": { + "key": "Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk" + } + } + }, + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "database": { + "id": "db", + "client": "client", + "databaseName": "db" + } + }, + { + "collection": { + "id": "coll", + "database": "db", + "collectionName": "coll" + } + } + ], + "initialData": [ + { + "databaseName": "keyvault", + "collectionName": "datakeys", + "documents": [ + { + "_id": { + "$binary": { + "base64": "q83vqxI0mHYSNBI0VniQEg==", + "subType": "04" + } + }, + "keyMaterial": { + "$binary": { + "base64": "HBk9BWihXExNDvTp1lUxOuxuZK2Pe2ZdVdlsxPEBkiO1bS4mG5NNDsQ7zVxJAH8BtdOYp72Ku4Y3nwc0BUpIKsvAKX4eYXtlhv5zUQxWdeNFhg9qK7qb8nqhnnLeT0f25jFSqzWJoT379hfwDeu0bebJHr35QrJ8myZdPMTEDYF08QYQ48ShRBli0S+QzBHHAQiM2iJNr4svg2WR8JSeWQ==", + "subType": "00" + } + }, + "creationDate": { + "$date": { + "$numberLong": "1648914851981" + } + }, + "updateDate": { + "$date": { + "$numberLong": "1648914851981" + } + }, + "status": { + "$numberInt": "0" + }, + "masterKey": { + "provider": "local" + } + } + ] + }, + { + "databaseName": "db", + "collectionName": "coll", + "documents": [], + "createOptions": { + "encryptedFields": { + "fields": [ + { + "keyId": { + "$binary": { + "base64": "q83vqxI0mHYSNBI0VniQEg==", + "subType": "04" + } + }, + "path": "encryptedText", + "bsonType": "string", + "queries": [ + { + "queryType": "suffixPreview", + "contention": { + "$numberLong": "0" + }, + "strMinQueryLength": { + "$numberLong": "3" + }, + "strMaxQueryLength": { + "$numberLong": "30" + }, + "caseSensitive": true, + "diacriticSensitive": true + } + ] + } + ] + } + } + } + ], + "tests": [ + { + "description": "Insert QE suffixPreview", + "operations": [ + { + "name": "insertOne", + "arguments": { + "document": { + "_id": 1, + "encryptedText": "foobar" + } + }, + "object": "coll" + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "command": { + "listCollections": 1, + "filter": { + "name": "coll" + } + }, + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "command": { + "find": "datakeys", + "filter": { + "$or": [ + { + "_id": { + "$in": [ + { + "$binary": { + "base64": "q83vqxI0mHYSNBI0VniQEg==", + "subType": "04" + } + } + ] + } + }, + { + "keyAltNames": { + "$in": [] + } + } + ] + }, + "$db": "keyvault", + "readConcern": { + "level": "majority" + } + }, + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "command": { + "insert": "coll", + "documents": [ + { + "_id": 1, + "encryptedText": { + "$$type": "binData" + } + } + ], + "ordered": true + }, + "commandName": "insert" + } + } + ] + } + ] + }, + { + "description": "Query with matching $encStrStartsWith", + "operations": [ + { + "name": "insertOne", + "arguments": { + "document": { + "_id": 1, + "encryptedText": "foobar" + } + }, + "object": "coll" + }, + { + "name": "find", + "arguments": { + "filter": { + "$expr": { + "$encStrEndsWith": { + "input": "$encryptedText", + "suffix": "bar" + } + } + } + }, + "object": "coll", + "expectResult": [ + { + "_id": { + "$numberInt": "1" + }, + "encryptedText": "foobar", + "__safeContent__": [ + { + "$binary": { + "base64": "wpaMBVDjL4bHf9EtSP52PJFzyNn1R19+iNI/hWtvzdk=", + "subType": "00" + } + }, + { + "$binary": { + "base64": "uDCWsucUsJemUP7pmeb+Kd8B9qupVzI8wnLFqX1rkiU=", + "subType": "00" + } + }, + { + "$binary": { + "base64": "W3E1x4bHZ8SEHFz4zwXM0G5Z5WSwBhnxE8x5/qdP6JM=", + "subType": "00" + } + }, + { + "$binary": { + "base64": "6g/TXVDDf6z+ntResIvTKWdmIy4ajQ1rhwdNZIiEG7A=", + "subType": "00" + } + }, + { + "$binary": { + "base64": "hU+u/T3D6dHDpT3d/v5AlgtRoAufCXCAyO2jQlgsnCw=", + "subType": "00" + } + }, + { + "$binary": { + "base64": "vrPnq0AtBIURNgNGA6HJL+5/p5SBWe+qz8505TRo/dE=", + "subType": "00" + } + }, + { + "$binary": { + "base64": "W5pylBxdv2soY2NcBfPiHDVLTS6tx+0ULkI8gysBeFY=", + "subType": "00" + } + }, + { + "$binary": { + "base64": "oWO3xX3x0bYUJGK2S1aPAmlU3Xtfsgb9lTZ6flGAlsg=", + "subType": "00" + } + }, + { + "$binary": { + "base64": "SjZGucTEUbdpd86O8yj1pyMyBOOKxvAQ9C8ngZ9C5UE=", + "subType": "00" + } + }, + { + "$binary": { + "base64": "CEaMZkxVDVbnXr+To0DOyvsva04UQkIYP3KtgYVVwf8=", + "subType": "00" + } + } + ] + } + ] + } + ] + }, + { + "description": "Query with non-matching $encStrEndsWith", + "operations": [ + { + "name": "insertOne", + "arguments": { + "document": { + "_id": 1, + "encryptedText": "foobar" + } + }, + "object": "coll" + }, + { + "name": "find", + "arguments": { + "filter": { + "$expr": { + "$encStrEndsWith": { + "input": "$encryptedText", + "suffix": "foo" + } + } + } + }, + "object": "coll", + "expectResult": [] + } + ] + } + ] +} diff --git a/test/unified_format.py b/test/unified_format.py index 375d845783..bc21464ab6 100644 --- a/test/unified_format.py +++ b/test/unified_format.py @@ -158,6 +158,14 @@ def is_run_on_requirement_satisfied(requirement): if req_csfle is True: min_version_satisfied = Version.from_string("4.2") <= server_version csfle_satisfied = _HAVE_PYMONGOCRYPT and min_version_satisfied + elif isinstance(req_csfle, dict) and "minLibmongocryptVersion" in req_csfle: + csfle_satisfied = False + req_version = req_csfle["minLibmongocryptVersion"] + if _HAVE_PYMONGOCRYPT: + from pymongocrypt import libmongocrypt_version + + if Version.from_string(libmongocrypt_version()) >= Version.from_string(req_version): + csfle_satisfied = True return ( topology_satisfied @@ -449,7 +457,7 @@ class UnifiedSpecTestMixinV1(IntegrationTest): a class attribute ``TEST_SPEC``. """ - SCHEMA_VERSION = Version.from_string("1.23") + SCHEMA_VERSION = Version.from_string("1.25") RUN_ON_LOAD_BALANCER = True TEST_SPEC: Any TEST_PATH = "" # This gets filled in by generate_test_classes @@ -1530,7 +1538,7 @@ class SpecTestBase(with_metaclass(UnifiedSpecTestMeta)): # type: ignore # Add "encryption" marker if the "csfle" runOnRequirement is set. for req in test_spec.get("runOnRequirements", []): - if req.get("csfle", False): + if "csfle" in req: base = pytest.mark.encryption(base) return base From ddf9508e15d25091ec8f4936ce12865723dc3f29 Mon Sep 17 00:00:00 2001 From: Shane Harvey Date: Fri, 22 Aug 2025 14:51:39 -0700 Subject: [PATCH 08/53] PYTHON-5510 Fix server selection log message for commitTransaction (#2503) --- pymongo/asynchronous/client_session.py | 3 +-- pymongo/synchronous/client_session.py | 5 +++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pymongo/asynchronous/client_session.py b/pymongo/asynchronous/client_session.py index c30fc6679f..be02295cea 100644 --- a/pymongo/asynchronous/client_session.py +++ b/pymongo/asynchronous/client_session.py @@ -167,7 +167,6 @@ WTimeoutError, ) from pymongo.helpers_shared import _RETRYABLE_ERROR_CODES -from pymongo.operations import _Op from pymongo.read_concern import ReadConcern from pymongo.read_preferences import ReadPreference, _ServerMode from pymongo.server_type import SERVER_TYPE @@ -868,7 +867,7 @@ async def func( return await self._finish_transaction(conn, command_name) return await self._client._retry_internal( - func, self, None, retryable=True, operation=_Op.ABORT + func, self, None, retryable=True, operation=command_name ) async def _finish_transaction(self, conn: AsyncConnection, command_name: str) -> dict[str, Any]: diff --git a/pymongo/synchronous/client_session.py b/pymongo/synchronous/client_session.py index 68a01dd7e7..72a5b8e885 100644 --- a/pymongo/synchronous/client_session.py +++ b/pymongo/synchronous/client_session.py @@ -165,7 +165,6 @@ WTimeoutError, ) from pymongo.helpers_shared import _RETRYABLE_ERROR_CODES -from pymongo.operations import _Op from pymongo.read_concern import ReadConcern from pymongo.read_preferences import ReadPreference, _ServerMode from pymongo.server_type import SERVER_TYPE @@ -864,7 +863,9 @@ def func( ) -> dict[str, Any]: return self._finish_transaction(conn, command_name) - return self._client._retry_internal(func, self, None, retryable=True, operation=_Op.ABORT) + return self._client._retry_internal( + func, self, None, retryable=True, operation=command_name + ) def _finish_transaction(self, conn: Connection, command_name: str) -> dict[str, Any]: self._transaction.attempt += 1 From 3ebd93480a8e66be27163c86aa87f01762e55f4a Mon Sep 17 00:00:00 2001 From: Iris <58442094+sleepyStick@users.noreply.github.com> Date: Mon, 25 Aug 2025 08:54:10 -0700 Subject: [PATCH 09/53] PYTHON-5514 Specific assertions for "is" and "is not None" (#2502) --- test/asynchronous/test_collection.py | 2 +- test/asynchronous/test_session.py | 6 +++--- test/test_collection.py | 2 +- test/test_session.py | 6 +++--- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/test/asynchronous/test_collection.py b/test/asynchronous/test_collection.py index 6a85b63960..90a0518532 100644 --- a/test/asynchronous/test_collection.py +++ b/test/asynchronous/test_collection.py @@ -1319,7 +1319,7 @@ async def test_error_code(self): self.assertIn(exc.code, (9, 10147, 16840, 17009)) # Just check that we set the error document. Fields # vary by MongoDB version. - self.assertTrue(exc.details is not None) + self.assertIsNotNone(exc.details) else: self.fail("OperationFailure was not raised") diff --git a/test/asynchronous/test_session.py b/test/asynchronous/test_session.py index 5ed3597751..19ce868c56 100644 --- a/test/asynchronous/test_session.py +++ b/test/asynchronous/test_session.py @@ -378,9 +378,9 @@ async def test_cursor_clone(self): async with self.client.start_session() as s: cursor = coll.find(session=s) - self.assertTrue(cursor.session is s) + self.assertIs(cursor.session, s) clone = cursor.clone() - self.assertTrue(clone.session is s) + self.assertIs(clone.session, s) # No explicit session. cursor = coll.find(batch_size=2) @@ -392,7 +392,7 @@ async def test_cursor_clone(self): await anext(clone) self.assertIsNone(clone.session) self.assertIsNotNone(clone._session) - self.assertFalse(cursor._session is clone._session) + self.assertIsNot(cursor._session, clone._session) await cursor.close() await clone.close() diff --git a/test/test_collection.py b/test/test_collection.py index 0dce88423b..b1947259ba 100644 --- a/test/test_collection.py +++ b/test/test_collection.py @@ -1305,7 +1305,7 @@ def test_error_code(self): self.assertIn(exc.code, (9, 10147, 16840, 17009)) # Just check that we set the error document. Fields # vary by MongoDB version. - self.assertTrue(exc.details is not None) + self.assertIsNotNone(exc.details) else: self.fail("OperationFailure was not raised") diff --git a/test/test_session.py b/test/test_session.py index 16a219ae52..40d0a53afb 100644 --- a/test/test_session.py +++ b/test/test_session.py @@ -378,9 +378,9 @@ def test_cursor_clone(self): with self.client.start_session() as s: cursor = coll.find(session=s) - self.assertTrue(cursor.session is s) + self.assertIs(cursor.session, s) clone = cursor.clone() - self.assertTrue(clone.session is s) + self.assertIs(clone.session, s) # No explicit session. cursor = coll.find(batch_size=2) @@ -392,7 +392,7 @@ def test_cursor_clone(self): next(clone) self.assertIsNone(clone.session) self.assertIsNotNone(clone._session) - self.assertFalse(cursor._session is clone._session) + self.assertIsNot(cursor._session, clone._session) cursor.close() clone.close() From cd4e5db997515dfddfe48404ed7cf03fc1267cc2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Aug 2025 11:57:02 -0500 Subject: [PATCH 10/53] Bump pyright from 1.1.403 to 1.1.404 (#2506) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index b7afbc5473..391de1bfe5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,7 +66,7 @@ pymongocrypt_source = [ perf = ["simplejson"] typing = [ "mypy==1.17.1", - "pyright==1.1.403", + "pyright==1.1.404", "typing_extensions", "pip" ] From 9892e1bbe970dd118f3168fd37077054d447882f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Aug 2025 11:57:35 -0500 Subject: [PATCH 11/53] Update coverage requirement from <=7.10.3,>=5 to >=5,<=7.10.5 (#2507) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 391de1bfe5..111136d08b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,7 +55,7 @@ gevent = ["gevent", "cffi>=2.0.0b1;python_version=='3.14'"] eventlet = ["eventlet"] coverage = [ "pytest-cov", - "coverage>=5,<=7.10.3" + "coverage>=5,<=7.10.5" ] mockupdb = [ "mockupdb@git+https://github.com/mongodb-labs/mongo-mockup-db@master" From 1179c5cdfbad5c3827c2f92284a09c016e043e87 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 25 Aug 2025 20:50:51 -0500 Subject: [PATCH 12/53] DRIVERS-3218 Avoid clearing the connection pool when the server connection rate limiter triggers --- pymongo/asynchronous/pool.py | 39 ++++++++++++++++++++++++++++---- pymongo/asynchronous/server.py | 4 ++++ pymongo/asynchronous/topology.py | 18 ++++++++++----- pymongo/synchronous/pool.py | 39 ++++++++++++++++++++++++++++---- pymongo/synchronous/server.py | 4 ++++ pymongo/synchronous/topology.py | 18 ++++++++++----- 6 files changed, 102 insertions(+), 20 deletions(-) diff --git a/pymongo/asynchronous/pool.py b/pymongo/asynchronous/pool.py index 8c169b4c52..6d41d93c19 100644 --- a/pymongo/asynchronous/pool.py +++ b/pymongo/asynchronous/pool.py @@ -37,7 +37,7 @@ from bson import DEFAULT_CODEC_OPTIONS from pymongo import _csot, helpers_shared from pymongo.asynchronous.client_session import _validate_session_write_concern -from pymongo.asynchronous.helpers import _handle_reauth +from pymongo.asynchronous.helpers import _backoff, _handle_reauth from pymongo.asynchronous.network import command from pymongo.common import ( MAX_BSON_SIZE, @@ -791,6 +791,7 @@ def __init__( self._max_connecting = self.opts.max_connecting self._pending = 0 self._client_id = client_id + self._backoff = 0 if self.enabled_for_cmap: assert self.opts._event_listeners is not None self.opts._event_listeners.publish_pool_created( @@ -846,6 +847,8 @@ async def _reset( async with self.size_cond: if self.closed: return + # Clear the backoff state. + self._backoff = 0 if self.opts.pause_enabled and pause and not self.opts.load_balanced: old_state, self.state = self.state, PoolState.PAUSED self.gen.inc(service_id) @@ -937,6 +940,12 @@ async def update_is_writable(self, is_writable: Optional[bool]) -> None: for _socket in self.conns: _socket.update_is_writable(self.is_writable) # type: ignore[arg-type] + async def backoff(self, service_id: Optional[ObjectId] = None) -> None: + # Mark the pool as in backoff. + # TODO: how to handle load balancers? + self._backoff += 1 + # TODO: emit a message. + async def reset( self, service_id: Optional[ObjectId] = None, interrupt_connections: bool = False ) -> None: @@ -994,7 +1003,8 @@ async def remove_stale_sockets(self, reference_generation: int) -> None: async with self._max_connecting_cond: # If maxConnecting connections are already being created # by this pool then try again later instead of waiting. - if self._pending >= self._max_connecting: + max_connecting = 1 if self._backoff else self._max_connecting + if self._pending >= max_connecting: return self._pending += 1 incremented = True @@ -1051,6 +1061,10 @@ async def connect(self, handler: Optional[_MongoClientErrorHandler] = None) -> A driverConnectionId=conn_id, ) + # Apply backoff if applicable. + if self._backoff: + await asyncio.sleep(_backoff(self._backoff)) + try: networking_interface = await _configured_protocol_interface(self.address, self.opts) # Catch KeyboardInterrupt, CancelledError, etc. and cleanup. @@ -1103,6 +1117,8 @@ async def connect(self, handler: Optional[_MongoClientErrorHandler] = None) -> A if handler: await handler.client._topology.receive_cluster_time(conn._cluster_time) + # Clear the backoff state. + self._backoff = 0 return conn @contextlib.asynccontextmanager @@ -1279,12 +1295,13 @@ async def _get_conn( # to be checked back into the pool. async with self._max_connecting_cond: self._raise_if_not_ready(checkout_started_time, emit_event=False) - while not (self.conns or self._pending < self._max_connecting): + max_connecting = 1 if self._backoff else self._max_connecting + while not (self.conns or self._pending < max_connecting): timeout = deadline - time.monotonic() if deadline else None if not await _async_cond_wait(self._max_connecting_cond, timeout): # Timed out, notify the next thread to ensure a # timeout doesn't consume the condition. - if self.conns or self._pending < self._max_connecting: + if self.conns or self._pending < max_connecting: self._max_connecting_cond.notify() emitted_event = True self._raise_wait_queue_timeout(checkout_started_time) @@ -1395,6 +1412,20 @@ async def checkin(self, conn: AsyncConnection) -> None: # Pool.reset(). if self.stale_generation(conn.generation, conn.service_id): close_conn = True + # If in backoff state, check the conn's readiness. + elif self._backoff: + # Set a 1ms read deadline and attempt to read 1 byte from the connection. + # Expect it to block for 1ms then return a deadline exceeded error. If it + # returns any other error, the connection is not usable, so return false. + # If it doesn't return an error and actually reads data, the connection is + # also not usable, so return false. + conn.conn.get_conn.settimeout(0.001) + close_conn = True + try: + conn.conn.get_conn.read() + except Exception as _: + # TODO: verify the exception + close_conn = False else: conn.update_last_checkin_time() conn.update_is_writable(bool(self.is_writable)) diff --git a/pymongo/asynchronous/server.py b/pymongo/asynchronous/server.py index f212306174..73c9575098 100644 --- a/pymongo/asynchronous/server.py +++ b/pymongo/asynchronous/server.py @@ -91,6 +91,10 @@ async def reset(self, service_id: Optional[ObjectId] = None) -> None: """Clear the connection pool.""" await self.pool.reset(service_id) + async def backoff(self, service_id: Optional[ObjectId] = None) -> None: + """Set the connection pool in backoff mode.""" + await self.pool.backoff(service_id) + async def close(self) -> None: """Clear the connection pool and stop the monitor. diff --git a/pymongo/asynchronous/topology.py b/pymongo/asynchronous/topology.py index 283aabc690..96efd3e752 100644 --- a/pymongo/asynchronous/topology.py +++ b/pymongo/asynchronous/topology.py @@ -896,12 +896,18 @@ async def _handle_error(self, address: _Address, err_ctx: _ErrorContext) -> None # ... MUST NOT request an immediate check of the server." if not self._settings.load_balanced: await self._process_change(ServerDescription(address, error=error)) - # Clear the pool. - await server.reset(service_id) - # "When a client marks a server Unknown from `Network error when - # reading or writing`_, clients MUST cancel the hello check on - # that server and close the current monitoring connection." - server._monitor.cancel_check() + + if err_ctx.completed_handshake: + # Clear the pool. + await server.reset(service_id) + # "When a client marks a server Unknown from `Network error when + # reading or writing`_, clients MUST cancel the hello check on + # that server and close the current monitoring connection." + server._monitor.cancel_check() + return + + # Set the pool into backoff mode. + await server.backoff(service_id) async def handle_error(self, address: _Address, err_ctx: _ErrorContext) -> None: """Handle an application error. diff --git a/pymongo/synchronous/pool.py b/pymongo/synchronous/pool.py index f35ca4d0fd..8ae3ff3d66 100644 --- a/pymongo/synchronous/pool.py +++ b/pymongo/synchronous/pool.py @@ -84,7 +84,7 @@ from pymongo.server_type import SERVER_TYPE from pymongo.socket_checker import SocketChecker from pymongo.synchronous.client_session import _validate_session_write_concern -from pymongo.synchronous.helpers import _handle_reauth +from pymongo.synchronous.helpers import _backoff, _handle_reauth from pymongo.synchronous.network import command if TYPE_CHECKING: @@ -789,6 +789,7 @@ def __init__( self._max_connecting = self.opts.max_connecting self._pending = 0 self._client_id = client_id + self._backoff = 0 if self.enabled_for_cmap: assert self.opts._event_listeners is not None self.opts._event_listeners.publish_pool_created( @@ -844,6 +845,8 @@ def _reset( with self.size_cond: if self.closed: return + # Clear the backoff state. + self._backoff = 0 if self.opts.pause_enabled and pause and not self.opts.load_balanced: old_state, self.state = self.state, PoolState.PAUSED self.gen.inc(service_id) @@ -935,6 +938,12 @@ def update_is_writable(self, is_writable: Optional[bool]) -> None: for _socket in self.conns: _socket.update_is_writable(self.is_writable) # type: ignore[arg-type] + def backoff(self, service_id: Optional[ObjectId] = None) -> None: + # Mark the pool as in backoff. + # TODO: how to handle load balancers? + self._backoff += 1 + # TODO: emit a message. + def reset( self, service_id: Optional[ObjectId] = None, interrupt_connections: bool = False ) -> None: @@ -990,7 +999,8 @@ def remove_stale_sockets(self, reference_generation: int) -> None: with self._max_connecting_cond: # If maxConnecting connections are already being created # by this pool then try again later instead of waiting. - if self._pending >= self._max_connecting: + max_connecting = 1 if self._backoff else self._max_connecting + if self._pending >= max_connecting: return self._pending += 1 incremented = True @@ -1047,6 +1057,10 @@ def connect(self, handler: Optional[_MongoClientErrorHandler] = None) -> Connect driverConnectionId=conn_id, ) + # Apply backoff if applicable. + if self._backoff: + asyncio.sleep(_backoff(self._backoff)) + try: networking_interface = _configured_socket_interface(self.address, self.opts) # Catch KeyboardInterrupt, CancelledError, etc. and cleanup. @@ -1099,6 +1113,8 @@ def connect(self, handler: Optional[_MongoClientErrorHandler] = None) -> Connect if handler: handler.client._topology.receive_cluster_time(conn._cluster_time) + # Clear the backoff state. + self._backoff = 0 return conn @contextlib.contextmanager @@ -1275,12 +1291,13 @@ def _get_conn( # to be checked back into the pool. with self._max_connecting_cond: self._raise_if_not_ready(checkout_started_time, emit_event=False) - while not (self.conns or self._pending < self._max_connecting): + max_connecting = 1 if self._backoff else self._max_connecting + while not (self.conns or self._pending < max_connecting): timeout = deadline - time.monotonic() if deadline else None if not _cond_wait(self._max_connecting_cond, timeout): # Timed out, notify the next thread to ensure a # timeout doesn't consume the condition. - if self.conns or self._pending < self._max_connecting: + if self.conns or self._pending < max_connecting: self._max_connecting_cond.notify() emitted_event = True self._raise_wait_queue_timeout(checkout_started_time) @@ -1391,6 +1408,20 @@ def checkin(self, conn: Connection) -> None: # Pool.reset(). if self.stale_generation(conn.generation, conn.service_id): close_conn = True + # If in backoff state, check the conn's readiness. + elif self._backoff: + # Set a 1ms read deadline and attempt to read 1 byte from the connection. + # Expect it to block for 1ms then return a deadline exceeded error. If it + # returns any other error, the connection is not usable, so return false. + # If it doesn't return an error and actually reads data, the connection is + # also not usable, so return false. + conn.conn.get_conn.settimeout(0.001) + close_conn = True + try: + conn.conn.get_conn.read() + except Exception as _: + # TODO: verify the exception + close_conn = False else: conn.update_last_checkin_time() conn.update_is_writable(bool(self.is_writable)) diff --git a/pymongo/synchronous/server.py b/pymongo/synchronous/server.py index f57420918b..cdc88ece64 100644 --- a/pymongo/synchronous/server.py +++ b/pymongo/synchronous/server.py @@ -91,6 +91,10 @@ def reset(self, service_id: Optional[ObjectId] = None) -> None: """Clear the connection pool.""" self.pool.reset(service_id) + def backoff(self, service_id: Optional[ObjectId] = None) -> None: + """Set the connection pool in backoff mode.""" + self.pool.backoff(service_id) + def close(self) -> None: """Clear the connection pool and stop the monitor. diff --git a/pymongo/synchronous/topology.py b/pymongo/synchronous/topology.py index a4ca0e6e0f..0fe0ad49d7 100644 --- a/pymongo/synchronous/topology.py +++ b/pymongo/synchronous/topology.py @@ -894,12 +894,18 @@ def _handle_error(self, address: _Address, err_ctx: _ErrorContext) -> None: # ... MUST NOT request an immediate check of the server." if not self._settings.load_balanced: self._process_change(ServerDescription(address, error=error)) - # Clear the pool. - server.reset(service_id) - # "When a client marks a server Unknown from `Network error when - # reading or writing`_, clients MUST cancel the hello check on - # that server and close the current monitoring connection." - server._monitor.cancel_check() + + if err_ctx.completed_handshake: + # Clear the pool. + server.reset(service_id) + # "When a client marks a server Unknown from `Network error when + # reading or writing`_, clients MUST cancel the hello check on + # that server and close the current monitoring connection." + server._monitor.cancel_check() + return + + # Set the pool into backoff mode. + server.backoff(service_id) def handle_error(self, address: _Address, err_ctx: _ErrorContext) -> None: """Handle an application error. From bc91967b4e3820a6a765c20361fb488b66932a51 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 25 Aug 2025 20:53:13 -0500 Subject: [PATCH 13/53] set to one byte --- pymongo/asynchronous/pool.py | 2 +- pymongo/synchronous/pool.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pymongo/asynchronous/pool.py b/pymongo/asynchronous/pool.py index 6d41d93c19..032b525f4a 100644 --- a/pymongo/asynchronous/pool.py +++ b/pymongo/asynchronous/pool.py @@ -1422,7 +1422,7 @@ async def checkin(self, conn: AsyncConnection) -> None: conn.conn.get_conn.settimeout(0.001) close_conn = True try: - conn.conn.get_conn.read() + conn.conn.get_conn.read(1) except Exception as _: # TODO: verify the exception close_conn = False diff --git a/pymongo/synchronous/pool.py b/pymongo/synchronous/pool.py index 8ae3ff3d66..92109e053a 100644 --- a/pymongo/synchronous/pool.py +++ b/pymongo/synchronous/pool.py @@ -1418,7 +1418,7 @@ def checkin(self, conn: Connection) -> None: conn.conn.get_conn.settimeout(0.001) close_conn = True try: - conn.conn.get_conn.read() + conn.conn.get_conn.read(1) except Exception as _: # TODO: verify the exception close_conn = False From 8c361be2190ed6252065c6aff7db89de728c6557 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 26 Aug 2025 08:24:30 -0500 Subject: [PATCH 14/53] Bump the actions group with 5 updates (#2505) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Steven Silvester --- .github/workflows/codeql.yml | 6 +++--- .github/workflows/dist.yml | 4 ++-- .github/workflows/test-python.yml | 34 +++++++++++++++---------------- .github/workflows/zizmor.yml | 4 ++-- 4 files changed, 24 insertions(+), 24 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index fd2808ea19..8dffe1fa7b 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -38,7 +38,7 @@ jobs: build-mode: none steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: ref: ${{ inputs.ref }} persist-credentials: false @@ -46,7 +46,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@51f77329afa6477de8c49fc9c7046c15b9a4e79d # v3 + uses: github/codeql-action/init@3c3833e0f8c1c83d449a7478aa59c036a9165498 # v3 with: languages: ${{ matrix.language }} build-mode: ${{ matrix.build-mode }} @@ -63,6 +63,6 @@ jobs: pip install -e . - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@51f77329afa6477de8c49fc9c7046c15b9a4e79d # v3 + uses: github/codeql-action/analyze@3c3833e0f8c1c83d449a7478aa59c036a9165498 # v3 with: category: "/language:${{matrix.language}}" diff --git a/.github/workflows/dist.yml b/.github/workflows/dist.yml index 14c253fe73..f5f8c20c88 100644 --- a/.github/workflows/dist.yml +++ b/.github/workflows/dist.yml @@ -45,7 +45,7 @@ jobs: steps: - name: Checkout pymongo - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: fetch-depth: 0 persist-credentials: false @@ -108,7 +108,7 @@ jobs: name: Make SDist runs-on: macos-13 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 with: fetch-depth: 0 persist-credentials: false diff --git a/.github/workflows/test-python.yml b/.github/workflows/test-python.yml index 11255f9e49..d55c0d7c7c 100644 --- a/.github/workflows/test-python.yml +++ b/.github/workflows/test-python.yml @@ -19,11 +19,11 @@ jobs: static: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 with: persist-credentials: false - name: Install uv - uses: astral-sh/setup-uv@e92bafb6253dcd438e0484186d7669ea7a8ca1cc # v5 + uses: astral-sh/setup-uv@4959332f0f014c5280e7eac8b70c90cb574c9f9b # v5 with: enable-cache: true python-version: "3.9" @@ -61,11 +61,11 @@ jobs: name: CPython ${{ matrix.python-version }}-${{ matrix.os }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 with: persist-credentials: false - name: Install uv - uses: astral-sh/setup-uv@e92bafb6253dcd438e0484186d7669ea7a8ca1cc # v5 + uses: astral-sh/setup-uv@4959332f0f014c5280e7eac8b70c90cb574c9f9b # v5 with: enable-cache: true python-version: ${{ matrix.python-version }} @@ -80,11 +80,11 @@ jobs: runs-on: ubuntu-latest name: DocTest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 with: persist-credentials: false - name: Install uv - uses: astral-sh/setup-uv@e92bafb6253dcd438e0484186d7669ea7a8ca1cc # v5 + uses: astral-sh/setup-uv@4959332f0f014c5280e7eac8b70c90cb574c9f9b # v5 with: enable-cache: true python-version: "3.9" @@ -105,11 +105,11 @@ jobs: name: Docs Checks runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 with: persist-credentials: false - name: Install uv - uses: astral-sh/setup-uv@e92bafb6253dcd438e0484186d7669ea7a8ca1cc # v5 + uses: astral-sh/setup-uv@4959332f0f014c5280e7eac8b70c90cb574c9f9b # v5 with: enable-cache: true python-version: "3.9" @@ -124,11 +124,11 @@ jobs: name: Link Check runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 with: persist-credentials: false - name: Install uv - uses: astral-sh/setup-uv@e92bafb6253dcd438e0484186d7669ea7a8ca1cc # v5 + uses: astral-sh/setup-uv@4959332f0f014c5280e7eac8b70c90cb574c9f9b # v5 with: enable-cache: true python-version: "3.9" @@ -146,11 +146,11 @@ jobs: matrix: python: ["3.9", "3.11"] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 with: persist-credentials: false - name: Install uv - uses: astral-sh/setup-uv@e92bafb6253dcd438e0484186d7669ea7a8ca1cc # v5 + uses: astral-sh/setup-uv@4959332f0f014c5280e7eac8b70c90cb574c9f9b # v5 with: enable-cache: true python-version: "${{matrix.python}}" @@ -167,7 +167,7 @@ jobs: runs-on: ubuntu-latest name: "Make an sdist" steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 with: persist-credentials: false - uses: actions/setup-python@v5 @@ -225,11 +225,11 @@ jobs: runs-on: ubuntu-latest name: Test using minimum dependencies and supported Python steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 with: persist-credentials: false - name: Install uv - uses: astral-sh/setup-uv@e92bafb6253dcd438e0484186d7669ea7a8ca1cc # v5 + uses: astral-sh/setup-uv@4959332f0f014c5280e7eac8b70c90cb574c9f9b # v5 with: python-version: '3.9' - id: setup-mongodb @@ -251,11 +251,11 @@ jobs: runs-on: ubuntu-latest name: Test async's minimum dependencies and Python steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 with: persist-credentials: false - name: Install uv - uses: astral-sh/setup-uv@e92bafb6253dcd438e0484186d7669ea7a8ca1cc # v5 + uses: astral-sh/setup-uv@4959332f0f014c5280e7eac8b70c90cb574c9f9b # v5 with: python-version: '3.9' - id: setup-mongodb diff --git a/.github/workflows/zizmor.yml b/.github/workflows/zizmor.yml index 8a2bccf931..e7a39fa39e 100644 --- a/.github/workflows/zizmor.yml +++ b/.github/workflows/zizmor.yml @@ -14,8 +14,8 @@ jobs: security-events: write steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: persist-credentials: false - name: Run zizmor 🌈 - uses: zizmorcore/zizmor-action@383d31df2eb66a2f42db98c9654bdc73231f3e3a + uses: zizmorcore/zizmor-action@7f2abfff7488a44086dba64ed2f5a9b431508079 From 0d4c84e86ff8ba7c1ee0e01a1bc2690de502c7e3 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 26 Aug 2025 09:52:09 -0500 Subject: [PATCH 15/53] PYTHON-5519 Clean up uv handling (#2510) --- .evergreen/run-tests.sh | 5 +---- justfile | 26 +++++++++++++++----------- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/.evergreen/run-tests.sh b/.evergreen/run-tests.sh index 2b7d856d41..a9f2ba2b5c 100755 --- a/.evergreen/run-tests.sh +++ b/.evergreen/run-tests.sh @@ -26,12 +26,9 @@ else fi # List the packages. -uv sync ${UV_ARGS} --reinstall +uv sync ${UV_ARGS} --reinstall --quiet uv pip list -# Ensure we go back to base environment after the test. -trap "uv sync" EXIT HUP - # Start the test runner. uv run ${UV_ARGS} .evergreen/scripts/run_tests.py "$@" diff --git a/justfile b/justfile index 74ebb48823..24da94a499 100644 --- a/justfile +++ b/justfile @@ -2,7 +2,7 @@ set shell := ["bash", "-c"] # Commonly used command segments. -uv_run := "uv run --isolated --frozen " +uv_run := "uv run --frozen " typing_run := uv_run + "--group typing --extra aws --extra encryption --extra ocsp --extra snappy --extra test --extra zstd" docs_run := uv_run + "--extra docs" doc_build := "./doc/_build" @@ -13,51 +13,55 @@ mypy_args := "--install-types --non-interactive" default: @just --list +[private] +resync: + @uv sync --quiet --frozen + install: bash .evergreen/scripts/setup-dev-env.sh [group('docs')] -docs: +docs: && resync {{docs_run}} sphinx-build -W -b html doc {{doc_build}}/html [group('docs')] -docs-serve: +docs-serve: && resync {{docs_run}} sphinx-autobuild -W -b html doc --watch ./pymongo --watch ./bson --watch ./gridfs {{doc_build}}/serve [group('docs')] -docs-linkcheck: +docs-linkcheck: && resync {{docs_run}} sphinx-build -E -b linkcheck doc {{doc_build}}/linkcheck [group('typing')] -typing: +typing: && resync just typing-mypy just typing-pyright [group('typing')] -typing-mypy: +typing-mypy: && resync {{typing_run}} mypy {{mypy_args}} bson gridfs tools pymongo {{typing_run}} mypy {{mypy_args}} --config-file mypy_test.ini test {{typing_run}} mypy {{mypy_args}} test/test_typing.py test/test_typing_strict.py [group('typing')] -typing-pyright: +typing-pyright: && resync {{typing_run}} pyright test/test_typing.py test/test_typing_strict.py {{typing_run}} pyright -p strict_pyrightconfig.json test/test_typing_strict.py [group('lint')] -lint: +lint: && resync {{uv_run}} pre-commit run --all-files [group('lint')] -lint-manual: +lint-manual: && resync {{uv_run}} pre-commit run --all-files --hook-stage manual [group('test')] -test *args="-v --durations=5 --maxfail=10": +test *args="-v --durations=5 --maxfail=10": && resync {{uv_run}} --extra test pytest {{args}} [group('test')] -run-tests *args: +run-tests *args: && resync bash ./.evergreen/run-tests.sh {{args}} [group('test')] From f51e8a5971fbe51a24bc4350e78603793b9f4f8a Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 26 Aug 2025 19:45:25 -0500 Subject: [PATCH 16/53] update approach --- pymongo/asynchronous/pool.py | 39 ++++++++++++-------------------- pymongo/asynchronous/server.py | 4 ---- pymongo/asynchronous/topology.py | 18 ++++++--------- pymongo/synchronous/pool.py | 39 ++++++++++++-------------------- pymongo/synchronous/server.py | 4 ---- pymongo/synchronous/topology.py | 18 ++++++--------- 6 files changed, 42 insertions(+), 80 deletions(-) diff --git a/pymongo/asynchronous/pool.py b/pymongo/asynchronous/pool.py index 032b525f4a..bec5e1b201 100644 --- a/pymongo/asynchronous/pool.py +++ b/pymongo/asynchronous/pool.py @@ -49,6 +49,7 @@ from pymongo.errors import ( # type:ignore[attr-defined] AutoReconnect, ConfigurationError, + ConnectionFailure, DocumentTooLarge, ExecutionTimeout, InvalidOperation, @@ -940,12 +941,6 @@ async def update_is_writable(self, is_writable: Optional[bool]) -> None: for _socket in self.conns: _socket.update_is_writable(self.is_writable) # type: ignore[arg-type] - async def backoff(self, service_id: Optional[ObjectId] = None) -> None: - # Mark the pool as in backoff. - # TODO: how to handle load balancers? - self._backoff += 1 - # TODO: emit a message. - async def reset( self, service_id: Optional[ObjectId] = None, interrupt_connections: bool = False ) -> None: @@ -1108,9 +1103,14 @@ async def connect(self, handler: Optional[_MongoClientErrorHandler] = None) -> A await conn.authenticate() # Catch KeyboardInterrupt, CancelledError, etc. and cleanup. - except BaseException: + except BaseException as e: async with self.lock: self.active_contexts.discard(conn.cancel_context) + # Enter backoff mode and reconnect on establishment failure. + if isinstance(e, ConnectionFailure): + self._backoff += 1 + # TODO: emit a message about backoff. + return await self.connect(handler) await conn.close_conn(ConnectionClosedReason.ERROR) raise @@ -1412,20 +1412,6 @@ async def checkin(self, conn: AsyncConnection) -> None: # Pool.reset(). if self.stale_generation(conn.generation, conn.service_id): close_conn = True - # If in backoff state, check the conn's readiness. - elif self._backoff: - # Set a 1ms read deadline and attempt to read 1 byte from the connection. - # Expect it to block for 1ms then return a deadline exceeded error. If it - # returns any other error, the connection is not usable, so return false. - # If it doesn't return an error and actually reads data, the connection is - # also not usable, so return false. - conn.conn.get_conn.settimeout(0.001) - close_conn = True - try: - conn.conn.get_conn.read(1) - except Exception as _: - # TODO: verify the exception - close_conn = False else: conn.update_last_checkin_time() conn.update_is_writable(bool(self.is_writable)) @@ -1456,8 +1442,8 @@ async def _perished(self, conn: AsyncConnection) -> bool: :class:`~pymongo.errors.AutoReconnect` exceptions on server hiccups, etc. We only check if the socket was closed by an external error if it has been > 1 second since the socket was checked into the - pool, to keep performance reasonable - we can't avoid AutoReconnects - completely anyway. + pool, or we are in backoff mode, to keep performance reasonable - + we can't avoid AutoReconnects completely anyway. """ idle_time_seconds = conn.idle_time_seconds() # If socket is idle, open a new one. @@ -1468,8 +1454,11 @@ async def _perished(self, conn: AsyncConnection) -> bool: await conn.close_conn(ConnectionClosedReason.IDLE) return True - if self._check_interval_seconds is not None and ( - self._check_interval_seconds == 0 or idle_time_seconds > self._check_interval_seconds + check_interval_seconds = self._check_interval_seconds + if self._backoff: + check_interval_seconds = 0 + if check_interval_seconds is not None and ( + check_interval_seconds == 0 or idle_time_seconds > check_interval_seconds ): if conn.conn_closed(): await conn.close_conn(ConnectionClosedReason.ERROR) diff --git a/pymongo/asynchronous/server.py b/pymongo/asynchronous/server.py index 73c9575098..f212306174 100644 --- a/pymongo/asynchronous/server.py +++ b/pymongo/asynchronous/server.py @@ -91,10 +91,6 @@ async def reset(self, service_id: Optional[ObjectId] = None) -> None: """Clear the connection pool.""" await self.pool.reset(service_id) - async def backoff(self, service_id: Optional[ObjectId] = None) -> None: - """Set the connection pool in backoff mode.""" - await self.pool.backoff(service_id) - async def close(self) -> None: """Clear the connection pool and stop the monitor. diff --git a/pymongo/asynchronous/topology.py b/pymongo/asynchronous/topology.py index 96efd3e752..fcb9af7ee7 100644 --- a/pymongo/asynchronous/topology.py +++ b/pymongo/asynchronous/topology.py @@ -897,17 +897,13 @@ async def _handle_error(self, address: _Address, err_ctx: _ErrorContext) -> None if not self._settings.load_balanced: await self._process_change(ServerDescription(address, error=error)) - if err_ctx.completed_handshake: - # Clear the pool. - await server.reset(service_id) - # "When a client marks a server Unknown from `Network error when - # reading or writing`_, clients MUST cancel the hello check on - # that server and close the current monitoring connection." - server._monitor.cancel_check() - return - - # Set the pool into backoff mode. - await server.backoff(service_id) + # Clear the pool. + await server.reset(service_id) + # "When a client marks a server Unknown from `Network error when + # reading or writing`_, clients MUST cancel the hello check on + # that server and close the current monitoring connection." + server._monitor.cancel_check() + return async def handle_error(self, address: _Address, err_ctx: _ErrorContext) -> None: """Handle an application error. diff --git a/pymongo/synchronous/pool.py b/pymongo/synchronous/pool.py index 92109e053a..7a529919d2 100644 --- a/pymongo/synchronous/pool.py +++ b/pymongo/synchronous/pool.py @@ -46,6 +46,7 @@ from pymongo.errors import ( # type:ignore[attr-defined] AutoReconnect, ConfigurationError, + ConnectionFailure, DocumentTooLarge, ExecutionTimeout, InvalidOperation, @@ -938,12 +939,6 @@ def update_is_writable(self, is_writable: Optional[bool]) -> None: for _socket in self.conns: _socket.update_is_writable(self.is_writable) # type: ignore[arg-type] - def backoff(self, service_id: Optional[ObjectId] = None) -> None: - # Mark the pool as in backoff. - # TODO: how to handle load balancers? - self._backoff += 1 - # TODO: emit a message. - def reset( self, service_id: Optional[ObjectId] = None, interrupt_connections: bool = False ) -> None: @@ -1104,9 +1099,14 @@ def connect(self, handler: Optional[_MongoClientErrorHandler] = None) -> Connect conn.authenticate() # Catch KeyboardInterrupt, CancelledError, etc. and cleanup. - except BaseException: + except BaseException as e: with self.lock: self.active_contexts.discard(conn.cancel_context) + # Enter backoff mode and reconnect on establishment failure. + if isinstance(e, ConnectionFailure): + self._backoff += 1 + # TODO: emit a message about backoff. + return self.connect(handler) conn.close_conn(ConnectionClosedReason.ERROR) raise @@ -1408,20 +1408,6 @@ def checkin(self, conn: Connection) -> None: # Pool.reset(). if self.stale_generation(conn.generation, conn.service_id): close_conn = True - # If in backoff state, check the conn's readiness. - elif self._backoff: - # Set a 1ms read deadline and attempt to read 1 byte from the connection. - # Expect it to block for 1ms then return a deadline exceeded error. If it - # returns any other error, the connection is not usable, so return false. - # If it doesn't return an error and actually reads data, the connection is - # also not usable, so return false. - conn.conn.get_conn.settimeout(0.001) - close_conn = True - try: - conn.conn.get_conn.read(1) - except Exception as _: - # TODO: verify the exception - close_conn = False else: conn.update_last_checkin_time() conn.update_is_writable(bool(self.is_writable)) @@ -1452,8 +1438,8 @@ def _perished(self, conn: Connection) -> bool: :class:`~pymongo.errors.AutoReconnect` exceptions on server hiccups, etc. We only check if the socket was closed by an external error if it has been > 1 second since the socket was checked into the - pool, to keep performance reasonable - we can't avoid AutoReconnects - completely anyway. + pool, or we are in backoff mode, to keep performance reasonable - + we can't avoid AutoReconnects completely anyway. """ idle_time_seconds = conn.idle_time_seconds() # If socket is idle, open a new one. @@ -1464,8 +1450,11 @@ def _perished(self, conn: Connection) -> bool: conn.close_conn(ConnectionClosedReason.IDLE) return True - if self._check_interval_seconds is not None and ( - self._check_interval_seconds == 0 or idle_time_seconds > self._check_interval_seconds + check_interval_seconds = self._check_interval_seconds + if self._backoff: + check_interval_seconds = 0 + if check_interval_seconds is not None and ( + check_interval_seconds == 0 or idle_time_seconds > check_interval_seconds ): if conn.conn_closed(): conn.close_conn(ConnectionClosedReason.ERROR) diff --git a/pymongo/synchronous/server.py b/pymongo/synchronous/server.py index cdc88ece64..f57420918b 100644 --- a/pymongo/synchronous/server.py +++ b/pymongo/synchronous/server.py @@ -91,10 +91,6 @@ def reset(self, service_id: Optional[ObjectId] = None) -> None: """Clear the connection pool.""" self.pool.reset(service_id) - def backoff(self, service_id: Optional[ObjectId] = None) -> None: - """Set the connection pool in backoff mode.""" - self.pool.backoff(service_id) - def close(self) -> None: """Clear the connection pool and stop the monitor. diff --git a/pymongo/synchronous/topology.py b/pymongo/synchronous/topology.py index 0fe0ad49d7..57108ad832 100644 --- a/pymongo/synchronous/topology.py +++ b/pymongo/synchronous/topology.py @@ -895,17 +895,13 @@ def _handle_error(self, address: _Address, err_ctx: _ErrorContext) -> None: if not self._settings.load_balanced: self._process_change(ServerDescription(address, error=error)) - if err_ctx.completed_handshake: - # Clear the pool. - server.reset(service_id) - # "When a client marks a server Unknown from `Network error when - # reading or writing`_, clients MUST cancel the hello check on - # that server and close the current monitoring connection." - server._monitor.cancel_check() - return - - # Set the pool into backoff mode. - server.backoff(service_id) + # Clear the pool. + server.reset(service_id) + # "When a client marks a server Unknown from `Network error when + # reading or writing`_, clients MUST cancel the hello check on + # that server and close the current monitoring connection." + server._monitor.cancel_check() + return def handle_error(self, address: _Address, err_ctx: _ErrorContext) -> None: """Handle an application error. From 7584d2de2812a99829c2300f836310892aadfcea Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 26 Aug 2025 19:46:18 -0500 Subject: [PATCH 17/53] Revert "Merge branch 'master' of github.com:mongodb/mongo-python-driver into DRIVERS-3218" This reverts commit c1fe2e3743cc59647faf3027aaa442bf0a8f2e8b, reversing changes made to f51e8a5971fbe51a24bc4350e78603793b9f4f8a. --- .evergreen/remove-unimplemented-tests.sh | 5 + .evergreen/run-mongodb-oidc-test.sh | 2 +- .evergreen/run-tests.sh | 5 +- .github/workflows/codeql.yml | 6 +- .github/workflows/dist.yml | 4 +- .github/workflows/test-python.yml | 48 +- .github/workflows/zizmor.yml | 4 +- bson/decimal128.py | 39 -- doc/changelog.rst | 30 - justfile | 26 +- pymongo/asynchronous/client_session.py | 3 +- pymongo/synchronous/client_session.py | 5 +- pyproject.toml | 4 +- test/__init__.py | 5 +- test/asynchronous/__init__.py | 5 +- test/asynchronous/helpers.py | 248 +++++++- test/asynchronous/test_collection.py | 2 +- test/asynchronous/test_custom_types.py | 25 +- test/asynchronous/test_encryption.py | 46 +- test/asynchronous/test_session.py | 6 +- test/asynchronous/unified_format.py | 50 +- ...-Text-cleanupStructuredEncryptionData.json | 219 ------- ...-Text-compactStructuredEncryptionData.json | 261 --------- .../spec/unified/QE-Text-prefixPreview.json | 338 ----------- .../unified/QE-Text-substringPreview.json | 551 ------------------ .../spec/unified/QE-Text-suffixPreview.json | 338 ----------- .../unified/fle2v2-BypassQueryAnalysis.json | 322 ---------- ...EncryptedFields-vs-EncryptedFieldsMap.json | 256 -------- .../spec/unified/localSchema.json | 343 ----------- .../spec/unified/maxWireVersion.json | 101 ---- test/csot/deprecated-options.json | 67 +-- test/csot/global-timeoutMS.json | 20 +- test/csot/override-operation-timeoutMS.json | 40 +- test/csot/tailable-awaitData.json | 82 +-- test/helpers.py | 248 +++++++- test/helpers_shared.py | 271 --------- test/test_collection.py | 2 +- test/test_custom_types.py | 25 +- test/test_encryption.py | 46 +- test/test_session.py | 6 +- test/test_uri_spec.py | 2 +- .../valid-pass/poc-queryable-encryption.json | 193 ------ test/unified_format.py | 50 +- test/unified_format_shared.py | 15 +- 44 files changed, 696 insertions(+), 3668 deletions(-) delete mode 100644 test/client-side-encryption/spec/unified/QE-Text-cleanupStructuredEncryptionData.json delete mode 100644 test/client-side-encryption/spec/unified/QE-Text-compactStructuredEncryptionData.json delete mode 100644 test/client-side-encryption/spec/unified/QE-Text-prefixPreview.json delete mode 100644 test/client-side-encryption/spec/unified/QE-Text-substringPreview.json delete mode 100644 test/client-side-encryption/spec/unified/QE-Text-suffixPreview.json delete mode 100644 test/client-side-encryption/spec/unified/fle2v2-BypassQueryAnalysis.json delete mode 100644 test/client-side-encryption/spec/unified/fle2v2-EncryptedFields-vs-EncryptedFieldsMap.json delete mode 100644 test/client-side-encryption/spec/unified/localSchema.json delete mode 100644 test/client-side-encryption/spec/unified/maxWireVersion.json delete mode 100644 test/helpers_shared.py delete mode 100644 test/unified-test-format/valid-pass/poc-queryable-encryption.json diff --git a/.evergreen/remove-unimplemented-tests.sh b/.evergreen/remove-unimplemented-tests.sh index fd50010138..92685ab2b7 100755 --- a/.evergreen/remove-unimplemented-tests.sh +++ b/.evergreen/remove-unimplemented-tests.sh @@ -3,6 +3,11 @@ PYMONGO=$(dirname "$(cd "$(dirname "$0")" || exit; pwd)") rm $PYMONGO/test/transactions/legacy/errors-client.json # PYTHON-1894 rm $PYMONGO/test/connection_monitoring/wait-queue-fairness.json # PYTHON-1873 +rm $PYMONGO/test/client-side-encryption/spec/unified/fle2v2-BypassQueryAnalysis.json # PYTHON-5143 +rm $PYMONGO/test/client-side-encryption/spec/unified/fle2v2-EncryptedFields-vs-EncryptedFieldsMap.json # PYTHON-5143 +rm $PYMONGO/test/client-side-encryption/spec/unified/localSchema.json # PYTHON-5143 +rm $PYMONGO/test/client-side-encryption/spec/unified/maxWireVersion.json # PYTHON-5143 +rm $PYMONGO/test/unified-test-format/valid-pass/poc-queryable-encryption.json # PYTHON-5143 rm $PYMONGO/test/discovery_and_monitoring/unified/pool-clear-application-error.json # PYTHON-4918 rm $PYMONGO/test/discovery_and_monitoring/unified/pool-clear-checkout-error.json # PYTHON-4918 rm $PYMONGO/test/discovery_and_monitoring/unified/pool-clear-min-pool-size-error.json # PYTHON-4918 diff --git a/.evergreen/run-mongodb-oidc-test.sh b/.evergreen/run-mongodb-oidc-test.sh index b34013a6ac..1a1cd81a8b 100755 --- a/.evergreen/run-mongodb-oidc-test.sh +++ b/.evergreen/run-mongodb-oidc-test.sh @@ -8,7 +8,7 @@ if [ ${OIDC_ENV} == "k8s" ]; then SUB_TEST_NAME=$K8S_VARIANT-remote else SUB_TEST_NAME=$OIDC_ENV-remote - sudo apt-get install -y python3-dev build-essential + apt-get install -y python3-dev build-essential fi bash ./.evergreen/just.sh setup-tests auth_oidc $SUB_TEST_NAME diff --git a/.evergreen/run-tests.sh b/.evergreen/run-tests.sh index a9f2ba2b5c..2b7d856d41 100755 --- a/.evergreen/run-tests.sh +++ b/.evergreen/run-tests.sh @@ -26,9 +26,12 @@ else fi # List the packages. -uv sync ${UV_ARGS} --reinstall --quiet +uv sync ${UV_ARGS} --reinstall uv pip list +# Ensure we go back to base environment after the test. +trap "uv sync" EXIT HUP + # Start the test runner. uv run ${UV_ARGS} .evergreen/scripts/run_tests.py "$@" diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 8dffe1fa7b..fd2808ea19 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -38,7 +38,7 @@ jobs: build-mode: none steps: - name: Checkout repository - uses: actions/checkout@v5 + uses: actions/checkout@v4 with: ref: ${{ inputs.ref }} persist-credentials: false @@ -46,7 +46,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@3c3833e0f8c1c83d449a7478aa59c036a9165498 # v3 + uses: github/codeql-action/init@51f77329afa6477de8c49fc9c7046c15b9a4e79d # v3 with: languages: ${{ matrix.language }} build-mode: ${{ matrix.build-mode }} @@ -63,6 +63,6 @@ jobs: pip install -e . - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@3c3833e0f8c1c83d449a7478aa59c036a9165498 # v3 + uses: github/codeql-action/analyze@51f77329afa6477de8c49fc9c7046c15b9a4e79d # v3 with: category: "/language:${{matrix.language}}" diff --git a/.github/workflows/dist.yml b/.github/workflows/dist.yml index f5f8c20c88..14c253fe73 100644 --- a/.github/workflows/dist.yml +++ b/.github/workflows/dist.yml @@ -45,7 +45,7 @@ jobs: steps: - name: Checkout pymongo - uses: actions/checkout@v5 + uses: actions/checkout@v4 with: fetch-depth: 0 persist-credentials: false @@ -108,7 +108,7 @@ jobs: name: Make SDist runs-on: macos-13 steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v4 with: fetch-depth: 0 persist-credentials: false diff --git a/.github/workflows/test-python.yml b/.github/workflows/test-python.yml index d55c0d7c7c..96729e3a6e 100644 --- a/.github/workflows/test-python.yml +++ b/.github/workflows/test-python.yml @@ -19,16 +19,16 @@ jobs: static: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v4 with: persist-credentials: false + - name: Install just + uses: extractions/setup-just@e33e0265a09d6d736e2ee1e0eb685ef1de4669ff # v3 - name: Install uv - uses: astral-sh/setup-uv@4959332f0f014c5280e7eac8b70c90cb574c9f9b # v5 + uses: astral-sh/setup-uv@e92bafb6253dcd438e0484186d7669ea7a8ca1cc # v5 with: enable-cache: true python-version: "3.9" - - name: Install just - run: uv tool install rust-just - name: Install Python dependencies run: | just install @@ -61,11 +61,11 @@ jobs: name: CPython ${{ matrix.python-version }}-${{ matrix.os }} steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v4 with: persist-credentials: false - name: Install uv - uses: astral-sh/setup-uv@4959332f0f014c5280e7eac8b70c90cb574c9f9b # v5 + uses: astral-sh/setup-uv@e92bafb6253dcd438e0484186d7669ea7a8ca1cc # v5 with: enable-cache: true python-version: ${{ matrix.python-version }} @@ -80,16 +80,16 @@ jobs: runs-on: ubuntu-latest name: DocTest steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v4 with: persist-credentials: false + - name: Install just + uses: extractions/setup-just@e33e0265a09d6d736e2ee1e0eb685ef1de4669ff # v3 - name: Install uv - uses: astral-sh/setup-uv@4959332f0f014c5280e7eac8b70c90cb574c9f9b # v5 + uses: astral-sh/setup-uv@e92bafb6253dcd438e0484186d7669ea7a8ca1cc # v5 with: enable-cache: true python-version: "3.9" - - name: Install just - run: uv tool install rust-just - id: setup-mongodb uses: mongodb-labs/drivers-evergreen-tools@master with: @@ -105,16 +105,16 @@ jobs: name: Docs Checks runs-on: ubuntu-latest steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v4 with: persist-credentials: false - name: Install uv - uses: astral-sh/setup-uv@4959332f0f014c5280e7eac8b70c90cb574c9f9b # v5 + uses: astral-sh/setup-uv@e92bafb6253dcd438e0484186d7669ea7a8ca1cc # v5 with: enable-cache: true python-version: "3.9" - name: Install just - run: uv tool install rust-just + uses: extractions/setup-just@e33e0265a09d6d736e2ee1e0eb685ef1de4669ff # v3 - name: Install dependencies run: just install - name: Build docs @@ -124,16 +124,16 @@ jobs: name: Link Check runs-on: ubuntu-latest steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v4 with: persist-credentials: false - name: Install uv - uses: astral-sh/setup-uv@4959332f0f014c5280e7eac8b70c90cb574c9f9b # v5 + uses: astral-sh/setup-uv@e92bafb6253dcd438e0484186d7669ea7a8ca1cc # v5 with: enable-cache: true python-version: "3.9" - name: Install just - run: uv tool install rust-just + uses: extractions/setup-just@e33e0265a09d6d736e2ee1e0eb685ef1de4669ff # v3 - name: Install dependencies run: just install - name: Build docs @@ -146,16 +146,16 @@ jobs: matrix: python: ["3.9", "3.11"] steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v4 with: persist-credentials: false - name: Install uv - uses: astral-sh/setup-uv@4959332f0f014c5280e7eac8b70c90cb574c9f9b # v5 + uses: astral-sh/setup-uv@e92bafb6253dcd438e0484186d7669ea7a8ca1cc # v5 with: enable-cache: true python-version: "${{matrix.python}}" - name: Install just - run: uv tool install rust-just + uses: extractions/setup-just@e33e0265a09d6d736e2ee1e0eb685ef1de4669ff # v3 - name: Install dependencies run: | just install @@ -167,7 +167,7 @@ jobs: runs-on: ubuntu-latest name: "Make an sdist" steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v4 with: persist-credentials: false - uses: actions/setup-python@v5 @@ -225,11 +225,11 @@ jobs: runs-on: ubuntu-latest name: Test using minimum dependencies and supported Python steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v4 with: persist-credentials: false - name: Install uv - uses: astral-sh/setup-uv@4959332f0f014c5280e7eac8b70c90cb574c9f9b # v5 + uses: astral-sh/setup-uv@e92bafb6253dcd438e0484186d7669ea7a8ca1cc # v5 with: python-version: '3.9' - id: setup-mongodb @@ -251,11 +251,11 @@ jobs: runs-on: ubuntu-latest name: Test async's minimum dependencies and Python steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v4 with: persist-credentials: false - name: Install uv - uses: astral-sh/setup-uv@4959332f0f014c5280e7eac8b70c90cb574c9f9b # v5 + uses: astral-sh/setup-uv@e92bafb6253dcd438e0484186d7669ea7a8ca1cc # v5 with: python-version: '3.9' - id: setup-mongodb diff --git a/.github/workflows/zizmor.yml b/.github/workflows/zizmor.yml index e7a39fa39e..8a2bccf931 100644 --- a/.github/workflows/zizmor.yml +++ b/.github/workflows/zizmor.yml @@ -14,8 +14,8 @@ jobs: security-events: write steps: - name: Checkout repository - uses: actions/checkout@v5 + uses: actions/checkout@v4 with: persist-credentials: false - name: Run zizmor 🌈 - uses: zizmorcore/zizmor-action@7f2abfff7488a44086dba64ed2f5a9b431508079 + uses: zizmorcore/zizmor-action@383d31df2eb66a2f42db98c9654bdc73231f3e3a diff --git a/bson/decimal128.py b/bson/decimal128.py index 7480f94d0a..92c054d878 100644 --- a/bson/decimal128.py +++ b/bson/decimal128.py @@ -20,11 +20,8 @@ import decimal import struct -from decimal import Decimal from typing import Any, Sequence, Tuple, Type, Union -from bson.codec_options import TypeDecoder, TypeEncoder - _PACK_64 = struct.Struct(" Type[Decimal]: - return Decimal - - def transform_python(self, value: Any) -> Decimal128: - return Decimal128(value) - - -class DecimalDecoder(TypeDecoder): - """Converts BSON :class:`Decimal128` to Python :class:`decimal.Decimal`. - - For example:: - opts = CodecOptions(type_registry=TypeRegistry([DecimalDecoder()])) - bson.decode(data, codec_options=opts) - - .. versionadded:: 4.15 - """ - - @property - def bson_type(self) -> Type[Decimal128]: - return Decimal128 - - def transform_bson(self, value: Any) -> decimal.Decimal: - return value.to_decimal() - - def create_decimal128_context() -> decimal.Context: """Returns an instance of :class:`decimal.Context` appropriate for working with IEEE-754 128-bit decimal floating point values. diff --git a/doc/changelog.rst b/doc/changelog.rst index 305c989106..a553be0144 100644 --- a/doc/changelog.rst +++ b/doc/changelog.rst @@ -1,27 +1,5 @@ Changelog ========= -Changes in Version 4.15.0 (XXXX/XX/XX) --------------------------------------- -PyMongo 4.15 brings a number of changes including: - -- Added :class:`bson.decimal128.DecimalEncoder` and :class:`bson.decimal128.DecimalDecoder` - to support encoding and decoding of BSON Decimal128 values to decimal.Decimal values using the TypeRegistry API. - -Changes in Version 4.14.1 (2025/08/19) --------------------------------------- - -Version 4.14.1 is a bug fix release. - - - Fixed a bug in ``MongoClient.append_metadata()`` and ``AsyncMongoClient.append_metadata()`` - that allowed duplicate ``DriverInfo.name`` to be appended to the metadata. - -Issues Resolved -............... - -See the `PyMongo 4.14.1 release notes in JIRA`_ for the list of resolved issues -in this release. - -.. _PyMongo 4.14.1 release notes in JIRA: https://jira.mongodb.org/secure/ReleaseNote.jspa?projectId=10004&version=45256 Changes in Version 4.14.0 (2025/08/06) -------------------------------------- @@ -56,14 +34,6 @@ PyMongo 4.14 brings a number of changes including: - Changed :meth:`~pymongo.uri_parser.parse_uri`'s ``options`` return value to be type ``dict`` instead of ``_CaseInsensitiveDictionary``. -Issues Resolved -............... - -See the `PyMongo 4.14 release notes in JIRA`_ for the list of resolved issues -in this release. - -.. _PyMongo 4.14 release notes in JIRA: https://jira.mongodb.org/secure/ReleaseNote.jspa?projectId=10004&version=43041 - Changes in Version 4.13.2 (2025/06/17) -------------------------------------- diff --git a/justfile b/justfile index 24da94a499..74ebb48823 100644 --- a/justfile +++ b/justfile @@ -2,7 +2,7 @@ set shell := ["bash", "-c"] # Commonly used command segments. -uv_run := "uv run --frozen " +uv_run := "uv run --isolated --frozen " typing_run := uv_run + "--group typing --extra aws --extra encryption --extra ocsp --extra snappy --extra test --extra zstd" docs_run := uv_run + "--extra docs" doc_build := "./doc/_build" @@ -13,55 +13,51 @@ mypy_args := "--install-types --non-interactive" default: @just --list -[private] -resync: - @uv sync --quiet --frozen - install: bash .evergreen/scripts/setup-dev-env.sh [group('docs')] -docs: && resync +docs: {{docs_run}} sphinx-build -W -b html doc {{doc_build}}/html [group('docs')] -docs-serve: && resync +docs-serve: {{docs_run}} sphinx-autobuild -W -b html doc --watch ./pymongo --watch ./bson --watch ./gridfs {{doc_build}}/serve [group('docs')] -docs-linkcheck: && resync +docs-linkcheck: {{docs_run}} sphinx-build -E -b linkcheck doc {{doc_build}}/linkcheck [group('typing')] -typing: && resync +typing: just typing-mypy just typing-pyright [group('typing')] -typing-mypy: && resync +typing-mypy: {{typing_run}} mypy {{mypy_args}} bson gridfs tools pymongo {{typing_run}} mypy {{mypy_args}} --config-file mypy_test.ini test {{typing_run}} mypy {{mypy_args}} test/test_typing.py test/test_typing_strict.py [group('typing')] -typing-pyright: && resync +typing-pyright: {{typing_run}} pyright test/test_typing.py test/test_typing_strict.py {{typing_run}} pyright -p strict_pyrightconfig.json test/test_typing_strict.py [group('lint')] -lint: && resync +lint: {{uv_run}} pre-commit run --all-files [group('lint')] -lint-manual: && resync +lint-manual: {{uv_run}} pre-commit run --all-files --hook-stage manual [group('test')] -test *args="-v --durations=5 --maxfail=10": && resync +test *args="-v --durations=5 --maxfail=10": {{uv_run}} --extra test pytest {{args}} [group('test')] -run-tests *args: && resync +run-tests *args: bash ./.evergreen/run-tests.sh {{args}} [group('test')] diff --git a/pymongo/asynchronous/client_session.py b/pymongo/asynchronous/client_session.py index afd1937571..4bb927d995 100644 --- a/pymongo/asynchronous/client_session.py +++ b/pymongo/asynchronous/client_session.py @@ -169,6 +169,7 @@ WTimeoutError, ) from pymongo.helpers_shared import _RETRYABLE_ERROR_CODES +from pymongo.operations import _Op from pymongo.read_concern import ReadConcern from pymongo.read_preferences import ReadPreference, _ServerMode from pymongo.server_type import SERVER_TYPE @@ -877,7 +878,7 @@ async def func( return await self._finish_transaction(conn, command_name) return await self._client._retry_internal( - func, self, None, retryable=True, operation=command_name + func, self, None, retryable=True, operation=_Op.ABORT ) async def _finish_transaction(self, conn: AsyncConnection, command_name: str) -> dict[str, Any]: diff --git a/pymongo/synchronous/client_session.py b/pymongo/synchronous/client_session.py index 712048879c..a8f03fac74 100644 --- a/pymongo/synchronous/client_session.py +++ b/pymongo/synchronous/client_session.py @@ -166,6 +166,7 @@ WTimeoutError, ) from pymongo.helpers_shared import _RETRYABLE_ERROR_CODES +from pymongo.operations import _Op from pymongo.read_concern import ReadConcern from pymongo.read_preferences import ReadPreference, _ServerMode from pymongo.server_type import SERVER_TYPE @@ -872,9 +873,7 @@ def func( ) -> dict[str, Any]: return self._finish_transaction(conn, command_name) - return self._client._retry_internal( - func, self, None, retryable=True, operation=command_name - ) + return self._client._retry_internal(func, self, None, retryable=True, operation=_Op.ABORT) def _finish_transaction(self, conn: Connection, command_name: str) -> dict[str, Any]: self._transaction.attempt += 1 diff --git a/pyproject.toml b/pyproject.toml index 111136d08b..b7afbc5473 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,7 +55,7 @@ gevent = ["gevent", "cffi>=2.0.0b1;python_version=='3.14'"] eventlet = ["eventlet"] coverage = [ "pytest-cov", - "coverage>=5,<=7.10.5" + "coverage>=5,<=7.10.3" ] mockupdb = [ "mockupdb@git+https://github.com/mongodb-labs/mongo-mockup-db@master" @@ -66,7 +66,7 @@ pymongocrypt_source = [ perf = ["simplejson"] typing = [ "mypy==1.17.1", - "pyright==1.1.404", + "pyright==1.1.403", "typing_extensions", "pip" ] diff --git a/test/__init__.py b/test/__init__.py index 12660e3a4a..95c2d7ee9d 100644 --- a/test/__init__.py +++ b/test/__init__.py @@ -59,8 +59,7 @@ sys.path[0:0] = [""] -from test.helpers import client_knobs, global_knobs -from test.helpers_shared import ( +from test.helpers import ( COMPRESSORS, IS_SRV, MONGODB_API_VERSION, @@ -68,8 +67,10 @@ TEST_LOADBALANCER, TLS_OPTIONS, SystemCertsPatcher, + client_knobs, db_pwd, db_user, + global_knobs, host, is_server_resolvable, port, diff --git a/test/asynchronous/__init__.py b/test/asynchronous/__init__.py index 7b594b184d..96769dc9c5 100644 --- a/test/asynchronous/__init__.py +++ b/test/asynchronous/__init__.py @@ -59,8 +59,7 @@ sys.path[0:0] = [""] -from test.asynchronous.helpers import client_knobs, global_knobs -from test.helpers_shared import ( +from test.helpers import ( COMPRESSORS, IS_SRV, MONGODB_API_VERSION, @@ -68,8 +67,10 @@ TEST_LOADBALANCER, TLS_OPTIONS, SystemCertsPatcher, + client_knobs, db_pwd, db_user, + global_knobs, host, is_server_resolvable, port, diff --git a/test/asynchronous/helpers.py b/test/asynchronous/helpers.py index 892c629631..bcb004af51 100644 --- a/test/asynchronous/helpers.py +++ b/test/asynchronous/helpers.py @@ -12,22 +12,137 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Shared helper methods for pymongo, bson, and gridfs test suites.""" +"""Shared constants and helper methods for pymongo, bson, and gridfs test suites.""" from __future__ import annotations import asyncio +import base64 +import gc +import multiprocessing +import os +import signal +import socket +import subprocess +import sys import threading +import time import traceback -from functools import wraps -from typing import Optional, no_type_check +import unittest +import warnings +from inspect import iscoroutinefunction -from bson import SON -from pymongo import common from pymongo._asyncio_task import create_task + +try: + import ipaddress + + HAVE_IPADDRESS = True +except ImportError: + HAVE_IPADDRESS = False +from functools import wraps +from typing import Any, Callable, Dict, Generator, Optional, no_type_check +from unittest import SkipTest + +from bson.son import SON +from pymongo import common, message from pymongo.read_preferences import ReadPreference +from pymongo.ssl_support import HAVE_SSL, _ssl # type:ignore[attr-defined] +from pymongo.synchronous.uri_parser import parse_uri + +if HAVE_SSL: + import ssl _IS_SYNC = False +# Enable debug output for uncollectable objects. PyPy does not have set_debug. +if hasattr(gc, "set_debug"): + gc.set_debug( + gc.DEBUG_UNCOLLECTABLE | getattr(gc, "DEBUG_OBJECTS", 0) | getattr(gc, "DEBUG_INSTANCES", 0) + ) + +# The host and port of a single mongod or mongos, or the seed host +# for a replica set. +host = os.environ.get("DB_IP", "localhost") +port = int(os.environ.get("DB_PORT", 27017)) +IS_SRV = "mongodb+srv" in host + +db_user = os.environ.get("DB_USER", "user") +db_pwd = os.environ.get("DB_PASSWORD", "password") + +CERT_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), "certificates") +CLIENT_PEM = os.environ.get("CLIENT_PEM", os.path.join(CERT_PATH, "client.pem")) +CA_PEM = os.environ.get("CA_PEM", os.path.join(CERT_PATH, "ca.pem")) + +TLS_OPTIONS: Dict = {"tls": True} +if CLIENT_PEM: + TLS_OPTIONS["tlsCertificateKeyFile"] = CLIENT_PEM +if CA_PEM: + TLS_OPTIONS["tlsCAFile"] = CA_PEM + +COMPRESSORS = os.environ.get("COMPRESSORS") +MONGODB_API_VERSION = os.environ.get("MONGODB_API_VERSION") +TEST_LOADBALANCER = bool(os.environ.get("TEST_LOAD_BALANCER")) +SINGLE_MONGOS_LB_URI = os.environ.get("SINGLE_MONGOS_LB_URI") +MULTI_MONGOS_LB_URI = os.environ.get("MULTI_MONGOS_LB_URI") + +if TEST_LOADBALANCER: + res = parse_uri(SINGLE_MONGOS_LB_URI or "") + host, port = res["nodelist"][0] + db_user = res["username"] or db_user + db_pwd = res["password"] or db_pwd + + +# Shared KMS data. +LOCAL_MASTER_KEY = base64.b64decode( + b"Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ" + b"5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk" +) +AWS_CREDS = { + "accessKeyId": os.environ.get("FLE_AWS_KEY", ""), + "secretAccessKey": os.environ.get("FLE_AWS_SECRET", ""), +} +AWS_CREDS_2 = { + "accessKeyId": os.environ.get("FLE_AWS_KEY2", ""), + "secretAccessKey": os.environ.get("FLE_AWS_SECRET2", ""), +} +AZURE_CREDS = { + "tenantId": os.environ.get("FLE_AZURE_TENANTID", ""), + "clientId": os.environ.get("FLE_AZURE_CLIENTID", ""), + "clientSecret": os.environ.get("FLE_AZURE_CLIENTSECRET", ""), +} +GCP_CREDS = { + "email": os.environ.get("FLE_GCP_EMAIL", ""), + "privateKey": os.environ.get("FLE_GCP_PRIVATEKEY", ""), +} +KMIP_CREDS = {"endpoint": os.environ.get("FLE_KMIP_ENDPOINT", "localhost:5698")} + +# Ensure Evergreen metadata doesn't result in truncation +os.environ.setdefault("MONGOB_LOG_MAX_DOCUMENT_LENGTH", "2000") + + +def is_server_resolvable(): + """Returns True if 'server' is resolvable.""" + socket_timeout = socket.getdefaulttimeout() + socket.setdefaulttimeout(1) + try: + try: + socket.gethostbyname("server") + return True + except OSError: + return False + finally: + socket.setdefaulttimeout(socket_timeout) + + +def _create_user(authdb, user, pwd=None, roles=None, **kwargs): + cmd = SON([("createUser", user)]) + # X509 doesn't use a password + if pwd: + cmd["pwd"] = pwd + cmd["roles"] = roles or ["root"] + cmd.update(**kwargs) + return authdb.command(cmd) + async def async_repl_set_step_down(client, **kwargs): """Run replSetStepDown, first unfreezing a secondary with replSetFreeze.""" @@ -122,10 +237,133 @@ def __del__(self): raise Exception(msg) +def _all_users(db): + return {u["user"] for u in db.command("usersInfo").get("users", [])} + + +def sanitize_cmd(cmd): + cp = cmd.copy() + cp.pop("$clusterTime", None) + cp.pop("$db", None) + cp.pop("$readPreference", None) + cp.pop("lsid", None) + if MONGODB_API_VERSION: + # Stable API parameters + cp.pop("apiVersion", None) + # OP_MSG encoding may move the payload type one field to the + # end of the command. Do the same here. + name = next(iter(cp)) + try: + identifier = message._FIELD_MAP[name] + docs = cp.pop(identifier) + cp[identifier] = docs + except KeyError: + pass + return cp + + +def sanitize_reply(reply): + cp = reply.copy() + cp.pop("$clusterTime", None) + cp.pop("operationTime", None) + return cp + + +def print_thread_tracebacks() -> None: + """Print all Python thread tracebacks.""" + for thread_id, frame in sys._current_frames().items(): + sys.stderr.write(f"\n--- Traceback for thread {thread_id} ---\n") + traceback.print_stack(frame, file=sys.stderr) + + +def print_thread_stacks(pid: int) -> None: + """Print all C-level thread stacks for a given process id.""" + if sys.platform == "darwin": + cmd = ["lldb", "--attach-pid", f"{pid}", "--batch", "--one-line", '"thread backtrace all"'] + else: + cmd = ["gdb", f"--pid={pid}", "--batch", '--eval-command="thread apply all bt"'] + + try: + res = subprocess.run( + cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, encoding="utf-8" + ) + except Exception as exc: + sys.stderr.write(f"Could not print C-level thread stacks because {cmd[0]} failed: {exc}") + else: + sys.stderr.write(res.stdout) + + # Global knobs to speed up the test suite. global_knobs = client_knobs(events_queue_frequency=0.05) +def _get_executors(topology): + executors = [] + for server in topology._servers.values(): + # Some MockMonitor do not have an _executor. + if hasattr(server._monitor, "_executor"): + executors.append(server._monitor._executor) + if hasattr(server._monitor, "_rtt_monitor"): + executors.append(server._monitor._rtt_monitor._executor) + executors.append(topology._Topology__events_executor) + if topology._srv_monitor: + executors.append(topology._srv_monitor._executor) + + return [e for e in executors if e is not None] + + +def print_running_topology(topology): + running = [e for e in _get_executors(topology) if not e._stopped] + if running: + print( + "WARNING: found Topology with running threads:\n" + f" Threads: {running}\n" + f" Topology: {topology}\n" + f" Creation traceback:\n{topology._settings._stack}" + ) + + +def test_cases(suite): + """Iterator over all TestCases within a TestSuite.""" + for suite_or_case in suite._tests: + if isinstance(suite_or_case, unittest.TestCase): + # unittest.TestCase + yield suite_or_case + else: + # unittest.TestSuite + yield from test_cases(suite_or_case) + + +# Helper method to workaround https://bugs.python.org/issue21724 +def clear_warning_registry(): + """Clear the __warningregistry__ for all modules.""" + for _, module in list(sys.modules.items()): + if hasattr(module, "__warningregistry__"): + module.__warningregistry__ = {} # type:ignore[attr-defined] + + +class SystemCertsPatcher: + def __init__(self, ca_certs): + if ( + ssl.OPENSSL_VERSION.lower().startswith("libressl") + and sys.platform == "darwin" + and not _ssl.IS_PYOPENSSL + ): + raise SkipTest( + "LibreSSL on OSX doesn't support setting CA certificates " + "using SSL_CERT_FILE environment variable." + ) + self.original_certs = os.environ.get("SSL_CERT_FILE") + # Tell OpenSSL where CA certificates live. + os.environ["SSL_CERT_FILE"] = ca_certs + + def disable(self): + if self.original_certs is None: + os.environ.pop("SSL_CERT_FILE") + else: + os.environ["SSL_CERT_FILE"] = self.original_certs + + if _IS_SYNC: PARENT = threading.Thread else: diff --git a/test/asynchronous/test_collection.py b/test/asynchronous/test_collection.py index 90a0518532..6a85b63960 100644 --- a/test/asynchronous/test_collection.py +++ b/test/asynchronous/test_collection.py @@ -1319,7 +1319,7 @@ async def test_error_code(self): self.assertIn(exc.code, (9, 10147, 16840, 17009)) # Just check that we set the error document. Fields # vary by MongoDB version. - self.assertIsNotNone(exc.details) + self.assertTrue(exc.details is not None) else: self.fail("OperationFailure was not raised") diff --git a/test/asynchronous/test_custom_types.py b/test/asynchronous/test_custom_types.py index 82c54512cc..385f755a1d 100644 --- a/test/asynchronous/test_custom_types.py +++ b/test/asynchronous/test_custom_types.py @@ -23,7 +23,6 @@ from random import random from typing import Any, Tuple, Type, no_type_check -from bson.decimal128 import DecimalDecoder, DecimalEncoder from gridfs.asynchronous.grid_file import AsyncGridIn, AsyncGridOut sys.path[0:0] = [""] @@ -60,7 +59,29 @@ _IS_SYNC = False -DECIMAL_CODECOPTS = CodecOptions(type_registry=TypeRegistry([DecimalEncoder(), DecimalDecoder()])) +class DecimalEncoder(TypeEncoder): + @property + def python_type(self): + return Decimal + + def transform_python(self, value): + return Decimal128(value) + + +class DecimalDecoder(TypeDecoder): + @property + def bson_type(self): + return Decimal128 + + def transform_bson(self, value): + return value.to_decimal() + + +class DecimalCodec(DecimalDecoder, DecimalEncoder): + pass + + +DECIMAL_CODECOPTS = CodecOptions(type_registry=TypeRegistry([DecimalCodec()])) class UndecipherableInt64Type: diff --git a/test/asynchronous/test_encryption.py b/test/asynchronous/test_encryption.py index 337dba0f64..f6afa4b2a3 100644 --- a/test/asynchronous/test_encryption.py +++ b/test/asynchronous/test_encryption.py @@ -57,14 +57,11 @@ from test.asynchronous.test_bulk import AsyncBulkTestBase from test.asynchronous.unified_format import generate_test_classes from test.asynchronous.utils_spec_runner import AsyncSpecRunner -from test.helpers_shared import ( - ALL_KMS_PROVIDERS, +from test.helpers import ( AWS_CREDS, - AWS_TEMP_CREDS, AZURE_CREDS, CA_PEM, CLIENT_PEM, - DEFAULT_KMS_TLS, GCP_CREDS, KMIP_CREDS, LOCAL_MASTER_KEY, @@ -207,7 +204,7 @@ async def test_init_kms_tls_options(self): opts = AutoEncryptionOpts( {}, "k.d", - kms_tls_options=DEFAULT_KMS_TLS, + kms_tls_options={"kmip": {"tlsCAFile": CA_PEM, "tlsCertificateKeyFile": CLIENT_PEM}}, ) _kms_ssl_contexts = _parse_kms_tls_options(opts._kms_tls_options, _IS_SYNC) ctx = _kms_ssl_contexts["kmip"] @@ -619,10 +616,17 @@ async def test_with_statement(self): # Spec tests +AWS_TEMP_CREDS = { + "accessKeyId": os.environ.get("CSFLE_AWS_TEMP_ACCESS_KEY_ID", ""), + "secretAccessKey": os.environ.get("CSFLE_AWS_TEMP_SECRET_ACCESS_KEY", ""), + "sessionToken": os.environ.get("CSFLE_AWS_TEMP_SESSION_TOKEN", ""), +} + AWS_TEMP_NO_SESSION_CREDS = { "accessKeyId": os.environ.get("CSFLE_AWS_TEMP_ACCESS_KEY_ID", ""), "secretAccessKey": os.environ.get("CSFLE_AWS_TEMP_SECRET_ACCESS_KEY", ""), } +KMS_TLS_OPTS = {"kmip": {"tlsCAFile": CA_PEM, "tlsCertificateKeyFile": CLIENT_PEM}} class AsyncTestSpec(AsyncSpecRunner): @@ -659,7 +663,7 @@ def parse_auto_encrypt_opts(self, opts): self.skipTest("GCP environment credentials are not set") if "kmip" in kms_providers: kms_providers["kmip"] = KMIP_CREDS - opts["kms_tls_options"] = DEFAULT_KMS_TLS + opts["kms_tls_options"] = KMS_TLS_OPTS if "key_vault_namespace" not in opts: opts["key_vault_namespace"] = "keyvault.datakeys" if "extra_options" in opts: @@ -753,6 +757,14 @@ async def run_scenario(self): ) # Prose Tests +ALL_KMS_PROVIDERS = { + "aws": AWS_CREDS, + "azure": AZURE_CREDS, + "gcp": GCP_CREDS, + "kmip": KMIP_CREDS, + "local": {"key": LOCAL_MASTER_KEY}, +} + LOCAL_KEY_ID = Binary(base64.b64decode(b"LOCALAAAAAAAAAAAAAAAAA=="), UUID_SUBTYPE) AWS_KEY_ID = Binary(base64.b64decode(b"AWSAAAAAAAAAAAAAAAAAAA=="), UUID_SUBTYPE) AZURE_KEY_ID = Binary(base64.b64decode(b"AZUREAAAAAAAAAAAAAAAAA=="), UUID_SUBTYPE) @@ -839,17 +851,13 @@ async def asyncSetUp(self): self.KMS_PROVIDERS, "keyvault.datakeys", schema_map=schemas, - kms_tls_options=DEFAULT_KMS_TLS, + kms_tls_options=KMS_TLS_OPTS, ) self.client_encrypted = await self.async_rs_or_single_client( auto_encryption_opts=opts, uuidRepresentation="standard" ) self.client_encryption = self.create_client_encryption( - self.KMS_PROVIDERS, - "keyvault.datakeys", - self.client, - OPTS, - kms_tls_options=DEFAULT_KMS_TLS, + self.KMS_PROVIDERS, "keyvault.datakeys", self.client, OPTS, kms_tls_options=KMS_TLS_OPTS ) self.listener.reset() @@ -1058,7 +1066,7 @@ async def _test_corpus(self, opts): "keyvault.datakeys", async_client_context.client, OPTS, - kms_tls_options=DEFAULT_KMS_TLS, + kms_tls_options=KMS_TLS_OPTS, ) corpus = self.fix_up_curpus(json_data("corpus", "corpus.json")) @@ -1150,7 +1158,7 @@ async def _test_corpus(self, opts): async def test_corpus(self): opts = AutoEncryptionOpts( - self.kms_providers(), "keyvault.datakeys", kms_tls_options=DEFAULT_KMS_TLS + self.kms_providers(), "keyvault.datakeys", kms_tls_options=KMS_TLS_OPTS ) await self._test_corpus(opts) @@ -1161,7 +1169,7 @@ async def test_corpus_local_schema(self): self.kms_providers(), "keyvault.datakeys", schema_map=schemas, - kms_tls_options=DEFAULT_KMS_TLS, + kms_tls_options=KMS_TLS_OPTS, ) await self._test_corpus(opts) @@ -1292,7 +1300,7 @@ async def asyncSetUp(self): key_vault_namespace="keyvault.datakeys", key_vault_client=async_client_context.client, codec_options=OPTS, - kms_tls_options=DEFAULT_KMS_TLS, + kms_tls_options=KMS_TLS_OPTS, ) kms_providers_invalid = copy.deepcopy(kms_providers) @@ -1304,7 +1312,7 @@ async def asyncSetUp(self): key_vault_namespace="keyvault.datakeys", key_vault_client=async_client_context.client, codec_options=OPTS, - kms_tls_options=DEFAULT_KMS_TLS, + kms_tls_options=KMS_TLS_OPTS, ) self._kmip_host_error = None self._invalid_host_error = None @@ -2744,7 +2752,7 @@ async def run_test(self, src_provider, dst_provider): key_vault_client=self.client, key_vault_namespace="keyvault.datakeys", kms_providers=ALL_KMS_PROVIDERS, - kms_tls_options=DEFAULT_KMS_TLS, + kms_tls_options=KMS_TLS_OPTS, codec_options=OPTS, ) @@ -2764,7 +2772,7 @@ async def run_test(self, src_provider, dst_provider): key_vault_client=client2, key_vault_namespace="keyvault.datakeys", kms_providers=ALL_KMS_PROVIDERS, - kms_tls_options=DEFAULT_KMS_TLS, + kms_tls_options=KMS_TLS_OPTS, codec_options=OPTS, ) diff --git a/test/asynchronous/test_session.py b/test/asynchronous/test_session.py index 19ce868c56..5ed3597751 100644 --- a/test/asynchronous/test_session.py +++ b/test/asynchronous/test_session.py @@ -378,9 +378,9 @@ async def test_cursor_clone(self): async with self.client.start_session() as s: cursor = coll.find(session=s) - self.assertIs(cursor.session, s) + self.assertTrue(cursor.session is s) clone = cursor.clone() - self.assertIs(clone.session, s) + self.assertTrue(clone.session is s) # No explicit session. cursor = coll.find(batch_size=2) @@ -392,7 +392,7 @@ async def test_cursor_clone(self): await anext(clone) self.assertIsNone(clone.session) self.assertIsNotNone(clone._session) - self.assertIsNot(cursor._session, clone._session) + self.assertFalse(cursor._session is clone._session) await cursor.close() await clone.close() diff --git a/test/asynchronous/unified_format.py b/test/asynchronous/unified_format.py index 9bd0fabdb8..09bf7e83ea 100644 --- a/test/asynchronous/unified_format.py +++ b/test/asynchronous/unified_format.py @@ -37,7 +37,6 @@ ) from test.asynchronous.utils import async_get_pool, flaky from test.asynchronous.utils_spec_runner import SpecRunnerTask -from test.helpers_shared import ALL_KMS_PROVIDERS, DEFAULT_KMS_TLS from test.unified_format_shared import ( KMS_TLS_OPTS, PLACEHOLDER_MAP, @@ -62,8 +61,6 @@ from test.version import Version from typing import Any, Dict, List, Mapping, Optional -import pytest - import pymongo from bson import SON, json_util from bson.codec_options import DEFAULT_CODEC_OPTIONS @@ -79,7 +76,7 @@ from pymongo.asynchronous.encryption import AsyncClientEncryption from pymongo.asynchronous.helpers import anext from pymongo.driver_info import DriverInfo -from pymongo.encryption_options import _HAVE_PYMONGOCRYPT, AutoEncryptionOpts +from pymongo.encryption_options import _HAVE_PYMONGOCRYPT from pymongo.errors import ( AutoReconnect, BulkWriteError, @@ -159,14 +156,6 @@ async def is_run_on_requirement_satisfied(requirement): if req_csfle is True: min_version_satisfied = Version.from_string("4.2") <= server_version csfle_satisfied = _HAVE_PYMONGOCRYPT and min_version_satisfied - elif isinstance(req_csfle, dict) and "minLibmongocryptVersion" in req_csfle: - csfle_satisfied = False - req_version = req_csfle["minLibmongocryptVersion"] - if _HAVE_PYMONGOCRYPT: - from pymongocrypt import libmongocrypt_version - - if Version.from_string(libmongocrypt_version()) >= Version.from_string(req_version): - csfle_satisfied = True return ( topology_satisfied @@ -270,23 +259,6 @@ async def _create_entity(self, entity_spec, uri=None): kwargs: dict = {} observe_events = spec.get("observeEvents", []) - if "autoEncryptOpts" in spec: - auto_encrypt_opts = spec["autoEncryptOpts"].copy() - auto_encrypt_kwargs: dict = dict(kms_tls_options=DEFAULT_KMS_TLS) - kms_providers = ALL_KMS_PROVIDERS.copy() - key_vault_namespace = auto_encrypt_opts.pop("keyVaultNamespace") - for provider_name, provider_value in auto_encrypt_opts.pop("kmsProviders").items(): - kms_providers[provider_name].update(provider_value) - extra_opts = auto_encrypt_opts.pop("extraOptions", {}) - for key, value in extra_opts.items(): - auto_encrypt_kwargs[camel_to_snake(key)] = value - for key, value in auto_encrypt_opts.items(): - auto_encrypt_kwargs[camel_to_snake(key)] = value - auto_encryption_opts = AutoEncryptionOpts( - kms_providers, key_vault_namespace, **auto_encrypt_kwargs - ) - kwargs["auto_encryption_opts"] = auto_encryption_opts - # The unified tests use topologyOpeningEvent, we use topologyOpenedEvent for i in range(len(observe_events)): if "topologyOpeningEvent" == observe_events[i]: @@ -458,7 +430,7 @@ class UnifiedSpecTestMixinV1(AsyncIntegrationTest): a class attribute ``TEST_SPEC``. """ - SCHEMA_VERSION = Version.from_string("1.25") + SCHEMA_VERSION = Version.from_string("1.22") RUN_ON_LOAD_BALANCER = True TEST_SPEC: Any TEST_PATH = "" # This gets filled in by generate_test_classes @@ -490,13 +462,6 @@ async def insert_initial_data(self, initial_data): wc = WriteConcern(w="majority") else: wc = WriteConcern(w=1) - - # Remove any encryption collections associated with the collection. - collections = await db.list_collection_names() - for collection in collections: - if collection in [f"enxcol_.{coll_name}.esc", f"enxcol_.{coll_name}.ecoc"]: - await db.drop_collection(collection) - if documents: if opts: await db.create_collection(coll_name, **opts) @@ -599,6 +564,8 @@ def maybe_skip_test(self, spec): self.skipTest("CSOT not implemented for watch()") if "cursors" in class_name: self.skipTest("CSOT not implemented for cursors") + if "dropindex on collection" in description: + self.skipTest("PYTHON-5491") if ( "tailable" in class_name or "tailable" in description @@ -1549,14 +1516,7 @@ class SpecTestBase(with_metaclass(UnifiedSpecTestMeta)): # type: ignore TEST_SPEC = test_spec EXPECTED_FAILURES = expected_failures - base = SpecTestBase - - # Add "encryption" marker if the "csfle" runOnRequirement is set. - for req in test_spec.get("runOnRequirements", []): - if "csfle" in req: - base = pytest.mark.encryption(base) - - return base + return SpecTestBase for dirpath, _, filenames in os.walk(test_path): dirname = os.path.split(dirpath)[-1] diff --git a/test/client-side-encryption/spec/unified/QE-Text-cleanupStructuredEncryptionData.json b/test/client-side-encryption/spec/unified/QE-Text-cleanupStructuredEncryptionData.json deleted file mode 100644 index 24f33ab3ec..0000000000 --- a/test/client-side-encryption/spec/unified/QE-Text-cleanupStructuredEncryptionData.json +++ /dev/null @@ -1,219 +0,0 @@ -{ - "description": "QE-Text-cleanupStructuredEncryptionData", - "schemaVersion": "1.25", - "runOnRequirements": [ - { - "minServerVersion": "8.2.0", - "topologies": [ - "replicaset", - "sharded", - "load-balanced" - ], - "csfle": { - "minLibmongocryptVersion": "1.15.0" - } - } - ], - "createEntities": [ - { - "client": { - "id": "client", - "autoEncryptOpts": { - "keyVaultNamespace": "keyvault.datakeys", - "kmsProviders": { - "local": { - "key": "Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk" - } - } - }, - "observeEvents": [ - "commandStartedEvent" - ] - } - }, - { - "database": { - "id": "db", - "client": "client", - "databaseName": "db" - } - }, - { - "collection": { - "id": "coll", - "database": "db", - "collectionName": "coll" - } - } - ], - "initialData": [ - { - "databaseName": "keyvault", - "collectionName": "datakeys", - "documents": [ - { - "_id": { - "$binary": { - "base64": "q83vqxI0mHYSNBI0VniQEg==", - "subType": "04" - } - }, - "keyMaterial": { - "$binary": { - "base64": "HBk9BWihXExNDvTp1lUxOuxuZK2Pe2ZdVdlsxPEBkiO1bS4mG5NNDsQ7zVxJAH8BtdOYp72Ku4Y3nwc0BUpIKsvAKX4eYXtlhv5zUQxWdeNFhg9qK7qb8nqhnnLeT0f25jFSqzWJoT379hfwDeu0bebJHr35QrJ8myZdPMTEDYF08QYQ48ShRBli0S+QzBHHAQiM2iJNr4svg2WR8JSeWQ==", - "subType": "00" - } - }, - "creationDate": { - "$date": { - "$numberLong": "1648914851981" - } - }, - "updateDate": { - "$date": { - "$numberLong": "1648914851981" - } - }, - "status": { - "$numberInt": "0" - }, - "masterKey": { - "provider": "local" - } - } - ] - }, - { - "databaseName": "db", - "collectionName": "coll", - "documents": [], - "createOptions": { - "encryptedFields": { - "fields": [ - { - "keyId": { - "$binary": { - "base64": "q83vqxI0mHYSNBI0VniQEg==", - "subType": "04" - } - }, - "path": "encryptedText", - "bsonType": "string", - "queries": [ - { - "queryType": "suffixPreview", - "contention": { - "$numberLong": "0" - }, - "strMinQueryLength": { - "$numberLong": "3" - }, - "strMaxQueryLength": { - "$numberLong": "30" - }, - "caseSensitive": true, - "diacriticSensitive": true - } - ] - } - ] - } - } - } - ], - "tests": [ - { - "description": "QE Text cleanupStructuredEncryptionData works", - "operations": [ - { - "name": "runCommand", - "object": "db", - "arguments": { - "command": { - "cleanupStructuredEncryptionData": "coll" - }, - "commandName": "cleanupStructuredEncryptionData" - }, - "expectResult": { - "ok": 1 - } - } - ], - "expectEvents": [ - { - "client": "client", - "events": [ - { - "commandStartedEvent": { - "command": { - "listCollections": 1, - "filter": { - "name": "coll" - } - }, - "commandName": "listCollections" - } - }, - { - "commandStartedEvent": { - "command": { - "find": "datakeys", - "filter": { - "$or": [ - { - "_id": { - "$in": [ - { - "$binary": { - "base64": "q83vqxI0mHYSNBI0VniQEg==", - "subType": "04" - } - } - ] - } - }, - { - "keyAltNames": { - "$in": [] - } - } - ] - }, - "$db": "keyvault", - "readConcern": { - "level": "majority" - } - }, - "commandName": "find" - } - }, - { - "commandStartedEvent": { - "command": { - "cleanupStructuredEncryptionData": "coll", - "cleanupTokens": { - "encryptedText": { - "ecoc": { - "$binary": { - "base64": "SWO8WEoZ2r2Kx/muQKb7+COizy85nIIUFiHh4K9kcvA=", - "subType": "00" - } - }, - "anchorPaddingToken": { - "$binary": { - "base64": "YAiF7Iwhqq1UyfxPvm70xfQJtrIRPrjfD2yRLG1+saQ=", - "subType": "00" - } - } - } - } - }, - "commandName": "cleanupStructuredEncryptionData" - } - } - ] - } - ] - } - ] -} diff --git a/test/client-side-encryption/spec/unified/QE-Text-compactStructuredEncryptionData.json b/test/client-side-encryption/spec/unified/QE-Text-compactStructuredEncryptionData.json deleted file mode 100644 index c7abfe2d4b..0000000000 --- a/test/client-side-encryption/spec/unified/QE-Text-compactStructuredEncryptionData.json +++ /dev/null @@ -1,261 +0,0 @@ -{ - "description": "QE-Text-compactStructuredEncryptionData", - "schemaVersion": "1.25", - "runOnRequirements": [ - { - "minServerVersion": "8.2.0", - "topologies": [ - "replicaset", - "sharded", - "load-balanced" - ], - "csfle": { - "minLibmongocryptVersion": "1.15.0" - } - } - ], - "createEntities": [ - { - "client": { - "id": "client", - "autoEncryptOpts": { - "keyVaultNamespace": "keyvault.datakeys", - "kmsProviders": { - "local": { - "key": "Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk" - } - } - }, - "observeEvents": [ - "commandStartedEvent" - ] - } - }, - { - "database": { - "id": "db", - "client": "client", - "databaseName": "db" - } - }, - { - "collection": { - "id": "coll", - "database": "db", - "collectionName": "coll" - } - } - ], - "initialData": [ - { - "databaseName": "keyvault", - "collectionName": "datakeys", - "documents": [ - { - "_id": { - "$binary": { - "base64": "q83vqxI0mHYSNBI0VniQEg==", - "subType": "04" - } - }, - "keyMaterial": { - "$binary": { - "base64": "HBk9BWihXExNDvTp1lUxOuxuZK2Pe2ZdVdlsxPEBkiO1bS4mG5NNDsQ7zVxJAH8BtdOYp72Ku4Y3nwc0BUpIKsvAKX4eYXtlhv5zUQxWdeNFhg9qK7qb8nqhnnLeT0f25jFSqzWJoT379hfwDeu0bebJHr35QrJ8myZdPMTEDYF08QYQ48ShRBli0S+QzBHHAQiM2iJNr4svg2WR8JSeWQ==", - "subType": "00" - } - }, - "creationDate": { - "$date": { - "$numberLong": "1648914851981" - } - }, - "updateDate": { - "$date": { - "$numberLong": "1648914851981" - } - }, - "status": { - "$numberInt": "0" - }, - "masterKey": { - "provider": "local" - } - } - ] - }, - { - "databaseName": "db", - "collectionName": "coll", - "documents": [], - "createOptions": { - "encryptedFields": { - "fields": [ - { - "keyId": { - "$binary": { - "base64": "q83vqxI0mHYSNBI0VniQEg==", - "subType": "04" - } - }, - "path": "encryptedText", - "bsonType": "string", - "queries": [ - { - "queryType": "suffixPreview", - "contention": { - "$numberLong": "0" - }, - "strMinQueryLength": { - "$numberLong": "3" - }, - "strMaxQueryLength": { - "$numberLong": "30" - }, - "caseSensitive": true, - "diacriticSensitive": true - } - ] - } - ] - } - } - } - ], - "tests": [ - { - "description": "QE Text compactStructuredEncryptionData works", - "operations": [ - { - "name": "runCommand", - "object": "db", - "arguments": { - "command": { - "compactStructuredEncryptionData": "coll" - }, - "commandName": "compactStructuredEncryptionData" - }, - "expectResult": { - "ok": 1 - } - } - ], - "expectEvents": [ - { - "client": "client", - "events": [ - { - "commandStartedEvent": { - "command": { - "listCollections": 1, - "filter": { - "name": "coll" - } - }, - "commandName": "listCollections" - } - }, - { - "commandStartedEvent": { - "command": { - "find": "datakeys", - "filter": { - "$or": [ - { - "_id": { - "$in": [ - { - "$binary": { - "base64": "q83vqxI0mHYSNBI0VniQEg==", - "subType": "04" - } - } - ] - } - }, - { - "keyAltNames": { - "$in": [] - } - } - ] - }, - "$db": "keyvault", - "readConcern": { - "level": "majority" - } - }, - "commandName": "find" - } - }, - { - "commandStartedEvent": { - "command": { - "compactStructuredEncryptionData": "coll", - "encryptionInformation": { - "type": { - "$numberInt": "1" - }, - "schema": { - "db.coll": { - "fields": [ - { - "keyId": { - "$binary": { - "base64": "q83vqxI0mHYSNBI0VniQEg==", - "subType": "04" - } - }, - "path": "encryptedText", - "bsonType": "string", - "queries": [ - { - "queryType": "suffixPreview", - "contention": { - "$numberLong": "0" - }, - "strMinQueryLength": { - "$numberLong": "3" - }, - "strMaxQueryLength": { - "$numberLong": "30" - }, - "caseSensitive": true, - "diacriticSensitive": true - } - ] - } - ], - "strEncodeVersion": { - "$numberInt": "1" - }, - "escCollection": "enxcol_.coll.esc", - "ecocCollection": "enxcol_.coll.ecoc" - } - } - }, - "compactionTokens": { - "encryptedText": { - "ecoc": { - "$binary": { - "base64": "SWO8WEoZ2r2Kx/muQKb7+COizy85nIIUFiHh4K9kcvA=", - "subType": "00" - } - }, - "anchorPaddingToken": { - "$binary": { - "base64": "YAiF7Iwhqq1UyfxPvm70xfQJtrIRPrjfD2yRLG1+saQ=", - "subType": "00" - } - } - } - } - }, - "commandName": "compactStructuredEncryptionData" - } - } - ] - } - ] - } - ] -} diff --git a/test/client-side-encryption/spec/unified/QE-Text-prefixPreview.json b/test/client-side-encryption/spec/unified/QE-Text-prefixPreview.json deleted file mode 100644 index 7279385743..0000000000 --- a/test/client-side-encryption/spec/unified/QE-Text-prefixPreview.json +++ /dev/null @@ -1,338 +0,0 @@ -{ - "description": "QE-Text-prefixPreview", - "schemaVersion": "1.25", - "runOnRequirements": [ - { - "minServerVersion": "8.2.0", - "topologies": [ - "replicaset", - "sharded", - "load-balanced" - ], - "csfle": { - "minLibmongocryptVersion": "1.15.0" - } - } - ], - "createEntities": [ - { - "client": { - "id": "client", - "autoEncryptOpts": { - "keyVaultNamespace": "keyvault.datakeys", - "kmsProviders": { - "local": { - "key": "Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk" - } - } - }, - "observeEvents": [ - "commandStartedEvent" - ] - } - }, - { - "database": { - "id": "db", - "client": "client", - "databaseName": "db" - } - }, - { - "collection": { - "id": "coll", - "database": "db", - "collectionName": "coll" - } - } - ], - "initialData": [ - { - "databaseName": "keyvault", - "collectionName": "datakeys", - "documents": [ - { - "_id": { - "$binary": { - "base64": "q83vqxI0mHYSNBI0VniQEg==", - "subType": "04" - } - }, - "keyMaterial": { - "$binary": { - "base64": "HBk9BWihXExNDvTp1lUxOuxuZK2Pe2ZdVdlsxPEBkiO1bS4mG5NNDsQ7zVxJAH8BtdOYp72Ku4Y3nwc0BUpIKsvAKX4eYXtlhv5zUQxWdeNFhg9qK7qb8nqhnnLeT0f25jFSqzWJoT379hfwDeu0bebJHr35QrJ8myZdPMTEDYF08QYQ48ShRBli0S+QzBHHAQiM2iJNr4svg2WR8JSeWQ==", - "subType": "00" - } - }, - "creationDate": { - "$date": { - "$numberLong": "1648914851981" - } - }, - "updateDate": { - "$date": { - "$numberLong": "1648914851981" - } - }, - "status": { - "$numberInt": "0" - }, - "masterKey": { - "provider": "local" - } - } - ] - }, - { - "databaseName": "db", - "collectionName": "coll", - "documents": [], - "createOptions": { - "encryptedFields": { - "fields": [ - { - "keyId": { - "$binary": { - "base64": "q83vqxI0mHYSNBI0VniQEg==", - "subType": "04" - } - }, - "path": "encryptedText", - "bsonType": "string", - "queries": [ - { - "queryType": "prefixPreview", - "contention": { - "$numberLong": "0" - }, - "strMinQueryLength": { - "$numberLong": "3" - }, - "strMaxQueryLength": { - "$numberLong": "30" - }, - "caseSensitive": true, - "diacriticSensitive": true - } - ] - } - ] - } - } - } - ], - "tests": [ - { - "description": "Insert QE prefixPreview", - "operations": [ - { - "name": "insertOne", - "arguments": { - "document": { - "_id": 1, - "encryptedText": "foobar" - } - }, - "object": "coll" - } - ], - "expectEvents": [ - { - "client": "client", - "events": [ - { - "commandStartedEvent": { - "command": { - "listCollections": 1, - "filter": { - "name": "coll" - } - }, - "commandName": "listCollections" - } - }, - { - "commandStartedEvent": { - "command": { - "find": "datakeys", - "filter": { - "$or": [ - { - "_id": { - "$in": [ - { - "$binary": { - "base64": "q83vqxI0mHYSNBI0VniQEg==", - "subType": "04" - } - } - ] - } - }, - { - "keyAltNames": { - "$in": [] - } - } - ] - }, - "$db": "keyvault", - "readConcern": { - "level": "majority" - } - }, - "commandName": "find" - } - }, - { - "commandStartedEvent": { - "command": { - "insert": "coll", - "documents": [ - { - "_id": 1, - "encryptedText": { - "$$type": "binData" - } - } - ], - "ordered": true - }, - "commandName": "insert" - } - } - ] - } - ] - }, - { - "description": "Query with matching $encStrStartsWith", - "operations": [ - { - "name": "insertOne", - "arguments": { - "document": { - "_id": 1, - "encryptedText": "foobar" - } - }, - "object": "coll" - }, - { - "name": "find", - "arguments": { - "filter": { - "$expr": { - "$encStrStartsWith": { - "input": "$encryptedText", - "prefix": "foo" - } - } - } - }, - "object": "coll", - "expectResult": [ - { - "_id": { - "$numberInt": "1" - }, - "encryptedText": "foobar", - "__safeContent__": [ - { - "$binary": { - "base64": "wpaMBVDjL4bHf9EtSP52PJFzyNn1R19+iNI/hWtvzdk=", - "subType": "00" - } - }, - { - "$binary": { - "base64": "fmUMXTMV/XRiN0IL3VXxSEn6SQG9E6Po30kJKB8JJlQ=", - "subType": "00" - } - }, - { - "$binary": { - "base64": "vZIDMiFDgjmLNYVrrbnq1zT4hg7sGpe/PMtighSsnRc=", - "subType": "00" - } - }, - { - "$binary": { - "base64": "26Z5G+sHTzV3D7F8Y0m08389USZ2afinyFV3ez9UEBQ=", - "subType": "00" - } - }, - { - "$binary": { - "base64": "q/JEq8of7bE0QE5Id0XuOsNQ4qVpANYymcPQDUL2Ywk=", - "subType": "00" - } - }, - { - "$binary": { - "base64": "Uvvv46LkfbgLoPqZ6xTBzpgoYRTM6FUgRdqZ9eaVojI=", - "subType": "00" - } - }, - { - "$binary": { - "base64": "nMxdq2lladuBJA3lv3JC2MumIUtRJBNJVLp3PVE6nQk=", - "subType": "00" - } - }, - { - "$binary": { - "base64": "hS3V0qq5CF/SkTl3ZWWWgXcAJ8G5yGtkY2RwcHNc5Oc=", - "subType": "00" - } - }, - { - "$binary": { - "base64": "McgwYUxfKj5+4D0vskZymy4KA82s71MR25iV/Enutww=", - "subType": "00" - } - }, - { - "$binary": { - "base64": "Ciqdk1b+t+Vrr6oIlFFk0Zdym5BPmwN3glQ0/VcsVdM=", - "subType": "00" - } - } - ] - } - ] - } - ] - }, - { - "description": "Query with non-matching $encStrStartsWith", - "operations": [ - { - "name": "insertOne", - "arguments": { - "document": { - "_id": 1, - "encryptedText": "foobar" - } - }, - "object": "coll" - }, - { - "name": "find", - "arguments": { - "filter": { - "$expr": { - "$encStrStartsWith": { - "input": "$encryptedText", - "prefix": "bar" - } - } - } - }, - "object": "coll", - "expectResult": [] - } - ] - } - ] -} diff --git a/test/client-side-encryption/spec/unified/QE-Text-substringPreview.json b/test/client-side-encryption/spec/unified/QE-Text-substringPreview.json deleted file mode 100644 index 6a8f133eac..0000000000 --- a/test/client-side-encryption/spec/unified/QE-Text-substringPreview.json +++ /dev/null @@ -1,551 +0,0 @@ -{ - "description": "QE-Text-substringPreview", - "schemaVersion": "1.25", - "runOnRequirements": [ - { - "minServerVersion": "8.2.0", - "topologies": [ - "replicaset", - "sharded", - "load-balanced" - ], - "csfle": { - "minLibmongocryptVersion": "1.15.0" - } - } - ], - "createEntities": [ - { - "client": { - "id": "client", - "autoEncryptOpts": { - "keyVaultNamespace": "keyvault.datakeys", - "kmsProviders": { - "local": { - "key": "Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk" - } - } - }, - "observeEvents": [ - "commandStartedEvent" - ] - } - }, - { - "database": { - "id": "db", - "client": "client", - "databaseName": "db" - } - }, - { - "collection": { - "id": "coll", - "database": "db", - "collectionName": "coll" - } - } - ], - "initialData": [ - { - "databaseName": "keyvault", - "collectionName": "datakeys", - "documents": [ - { - "_id": { - "$binary": { - "base64": "q83vqxI0mHYSNBI0VniQEg==", - "subType": "04" - } - }, - "keyMaterial": { - "$binary": { - "base64": "HBk9BWihXExNDvTp1lUxOuxuZK2Pe2ZdVdlsxPEBkiO1bS4mG5NNDsQ7zVxJAH8BtdOYp72Ku4Y3nwc0BUpIKsvAKX4eYXtlhv5zUQxWdeNFhg9qK7qb8nqhnnLeT0f25jFSqzWJoT379hfwDeu0bebJHr35QrJ8myZdPMTEDYF08QYQ48ShRBli0S+QzBHHAQiM2iJNr4svg2WR8JSeWQ==", - "subType": "00" - } - }, - "creationDate": { - "$date": { - "$numberLong": "1648914851981" - } - }, - "updateDate": { - "$date": { - "$numberLong": "1648914851981" - } - }, - "status": { - "$numberInt": "0" - }, - "masterKey": { - "provider": "local" - } - } - ] - }, - { - "databaseName": "db", - "collectionName": "coll", - "documents": [], - "createOptions": { - "encryptedFields": { - "fields": [ - { - "keyId": { - "$binary": { - "base64": "q83vqxI0mHYSNBI0VniQEg==", - "subType": "04" - } - }, - "path": "encryptedText", - "bsonType": "string", - "queries": [ - { - "queryType": "substringPreview", - "contention": { - "$numberLong": "0" - }, - "strMinQueryLength": { - "$numberLong": "3" - }, - "strMaxQueryLength": { - "$numberLong": "10" - }, - "strMaxLength": { - "$numberLong": "20" - }, - "caseSensitive": true, - "diacriticSensitive": true - } - ] - } - ] - } - } - } - ], - "tests": [ - { - "description": "Insert QE suffixPreview", - "operations": [ - { - "name": "insertOne", - "arguments": { - "document": { - "_id": 1, - "encryptedText": "foobar" - } - }, - "object": "coll" - } - ], - "expectEvents": [ - { - "client": "client", - "events": [ - { - "commandStartedEvent": { - "command": { - "listCollections": 1, - "filter": { - "name": "coll" - } - }, - "commandName": "listCollections" - } - }, - { - "commandStartedEvent": { - "command": { - "find": "datakeys", - "filter": { - "$or": [ - { - "_id": { - "$in": [ - { - "$binary": { - "base64": "q83vqxI0mHYSNBI0VniQEg==", - "subType": "04" - } - } - ] - } - }, - { - "keyAltNames": { - "$in": [] - } - } - ] - }, - "$db": "keyvault", - "readConcern": { - "level": "majority" - } - }, - "commandName": "find" - } - }, - { - "commandStartedEvent": { - "command": { - "insert": "coll", - "documents": [ - { - "_id": 1, - "encryptedText": { - "$$type": "binData" - } - } - ], - "ordered": true - }, - "commandName": "insert" - } - } - ] - } - ] - }, - { - "description": "Query with matching $encStrContains", - "operations": [ - { - "name": "insertOne", - "arguments": { - "document": { - "_id": 1, - "encryptedText": "foobar" - } - }, - "object": "coll" - }, - { - "name": "find", - "arguments": { - "filter": { - "$expr": { - "$encStrContains": { - "input": "$encryptedText", - "substring": "oba" - } - } - } - }, - "object": "coll", - "expectResult": [ - { - "_id": { - "$numberInt": "1" - }, - "encryptedText": "foobar", - "__safeContent__": [ - { - "$binary": { - "base64": "wpaMBVDjL4bHf9EtSP52PJFzyNn1R19+iNI/hWtvzdk=", - "subType": "00" - } - }, - { - "$binary": { - "base64": "IpY3x/jjm8j/74jAdUhgxdM5hk68zR0zv/lTKm/72Vg=", - "subType": "00" - } - }, - { - "$binary": { - "base64": "G+ky260C6QiOfIxKz14FmaMbAxvui1BKJO/TnLOHlGk=", - "subType": "00" - } - }, - { - "$binary": { - "base64": "7dv3gAKe9vwJMZmpB40pRCwRTmc7ds9UkGhxH8j084E=", - "subType": "00" - } - }, - { - "$binary": { - "base64": "o0V+Efn6x8XQdE80F1tztNaT3qxHjcsd9DOQ47BtmQk=", - "subType": "00" - } - }, - { - "$binary": { - "base64": "sJvrCjyVot7PIZFsdRehWFANKAj6fmBaj3FLbz/dZLE=", - "subType": "00" - } - }, - { - "$binary": { - "base64": "e98auxFmu02h5MfBIARk29MI7hSmvN3F9DaQ0xjqoEM=", - "subType": "00" - } - }, - { - "$binary": { - "base64": "US83krGNov/ezL6IhsY5eEOCxv1xUPDIEL/nmY0IKi0=", - "subType": "00" - } - }, - { - "$binary": { - "base64": "P2Aq5+OHZPG0CWIdmZvWq9c/18ZKVYW3vbxd+WU/TXU=", - "subType": "00" - } - }, - { - "$binary": { - "base64": "8AdPRPnSzcd5uhq4TZfNvNeF0XjLNVwAsJJMTtktw84=", - "subType": "00" - } - }, - { - "$binary": { - "base64": "9O6u/G51I4ZHFLhL4ZLuudbr0s202A2QnPfThmOXPhI=", - "subType": "00" - } - }, - { - "$binary": { - "base64": "N7AjYVyVlv6+lVSTM+cIxRL3SMgs3G5LgxSs+jrgDkI=", - "subType": "00" - } - }, - { - "$binary": { - "base64": "RbGF7dQbPGYQFd9DDO1hPz1UlLOJ77FAC6NsjGwJeos=", - "subType": "00" - } - }, - { - "$binary": { - "base64": "m7srHMgKm6kZwsNx8rc45pmw0/9Qro6xuQ8lZS3+RYk=", - "subType": "00" - } - }, - { - "$binary": { - "base64": "K75CNU3JyKFqZWPiIsVi4+n7DhYmcPl/nEhQ3d88mVI=", - "subType": "00" - } - }, - { - "$binary": { - "base64": "c7bwGpUZc/7JzEnMS7qQ/TPuXZyrmMihFaAV6zIqbZc=", - "subType": "00" - } - }, - { - "$binary": { - "base64": "rDvEdUgEk8u4Srt3ETokWs2FXcnyJaRGQ+NbkFwi2rQ=", - "subType": "00" - } - }, - { - "$binary": { - "base64": "VcdZj9zfveRBRlpCR2OYWau2+GokOFb73TE3gpElNiU=", - "subType": "00" - } - }, - { - "$binary": { - "base64": "eOa9o2xfA6OgkbYUxd6wQJicaeN6guhy2V66W3ALsaA=", - "subType": "00" - } - }, - { - "$binary": { - "base64": "1xGkJh+um70XiRd8lKLDtyHgDqrf7/59Mg7X0+KZh8k=", - "subType": "00" - } - }, - { - "$binary": { - "base64": "OSvllqHxycbcZN4phR6NDujY3ttA59o7nQJ6V9eJpX0=", - "subType": "00" - } - }, - { - "$binary": { - "base64": "ZTX1pyk8Vdw0BSbJx7GeJNcQf3tGKxbrrNSTqBqUWkg=", - "subType": "00" - } - }, - { - "$binary": { - "base64": "cn7V05zb5iXwYrePGMHztC+GRq+Tj8IMpRDraauPhSE=", - "subType": "00" - } - }, - { - "$binary": { - "base64": "E9bV9KyrZxHJSUmMg0HrDK4gGN+75ruelAnrM6hXQgY=", - "subType": "00" - } - }, - { - "$binary": { - "base64": "WrssTNmdgXoTGpbaF0JLRCGH6cDQuz1XEFNTy98nrb0=", - "subType": "00" - } - }, - { - "$binary": { - "base64": "jZmyOJP35dsxQ/OY5U4ISpVRIYr8iedNfcwZiKt29Qc=", - "subType": "00" - } - }, - { - "$binary": { - "base64": "d2mocORMbX9MX+/itAW8r1kxVw2/uii4vzXtc+2CIRQ=", - "subType": "00" - } - }, - { - "$binary": { - "base64": "JBnJy58eRPhDo3DuZvsHbvQDiHXxdtAx1Eif66k5SfA=", - "subType": "00" - } - }, - { - "$binary": { - "base64": "OjbDulC8s62v0pgweBSsQqtJjJBwH5JinfJpj7nVr+A=", - "subType": "00" - } - }, - { - "$binary": { - "base64": "85i7KT2GP9nSda3Gsil5LKubhq0LDtc22pxBxHpR+nE=", - "subType": "00" - } - }, - { - "$binary": { - "base64": "u9Fvsclwrs9lwIcMPV/fMZD7L3d5anSfJQVjQb9mgLg=", - "subType": "00" - } - }, - { - "$binary": { - "base64": "LZ32ttmLJGOIw9oFaUCn3Sx5uHPTYJPSFpeGRWNqlUc=", - "subType": "00" - } - }, - { - "$binary": { - "base64": "mMsZvGEePTqtl0FJAL/jAdyWNQIlpwN61YIlZsSIZ6s=", - "subType": "00" - } - }, - { - "$binary": { - "base64": "XZcu1a/ZGsIzAl3j4MXQlLo4v2p7kvIqRHtIQYFmL6k=", - "subType": "00" - } - }, - { - "$binary": { - "base64": "Zse27LinlYCEnX6iTmJceI33mEJxFb0LdPxp0RiMOaQ=", - "subType": "00" - } - }, - { - "$binary": { - "base64": "vOv2Hgb2/sBpnX9XwFbIN6yDxhjchwlmczUf82W2tp4=", - "subType": "00" - } - }, - { - "$binary": { - "base64": "oQxZ9A6j3x5j6x1Jqw/N9tpP4rfWMjcV3y+a3PkrL7c=", - "subType": "00" - } - }, - { - "$binary": { - "base64": "/D7ew3EijyUnmT22awVFspcuyo3JChJcDeCPwpljzVM=", - "subType": "00" - } - }, - { - "$binary": { - "base64": "BEmmwqyamt9X3bcWDld61P01zquy8fBHAXq3SHAPP0M=", - "subType": "00" - } - }, - { - "$binary": { - "base64": "wygD9/kAo1KsRvtr1v+9/lvqoWdKwgh6gDHvAQfXPPk=", - "subType": "00" - } - }, - { - "$binary": { - "base64": "pRTKgF/uksrF1c1AcfSTY6ZhqBKVud1vIztQ4/36SLs=", - "subType": "00" - } - }, - { - "$binary": { - "base64": "C4iUo8oNJsjJ37BqnBgIgSQpf99X2Bb4W5MZEAmakHU=", - "subType": "00" - } - }, - { - "$binary": { - "base64": "icoE53jIq6Fu/YGKUiSUTYyZ8xdiTQY9jJiGxVJObpw=", - "subType": "00" - } - }, - { - "$binary": { - "base64": "oubCwk0V6G2RFWtcOnYDU4uUBoXBrhBRi4nZgrYj9JY=", - "subType": "00" - } - }, - { - "$binary": { - "base64": "IyqhQ9nGhzEi5YW2W6v1kGU5DY2u2qSqbM/qXdLdWVU=", - "subType": "00" - } - } - ] - } - ] - } - ] - }, - { - "description": "Query with non-matching $encStrContains", - "operations": [ - { - "name": "insertOne", - "arguments": { - "document": { - "_id": 1, - "encryptedText": "foobar" - } - }, - "object": "coll" - }, - { - "name": "find", - "arguments": { - "filter": { - "$expr": { - "$encStrContains": { - "input": "$encryptedText", - "substring": "blah" - } - } - } - }, - "object": "coll", - "expectResult": [] - } - ] - } - ] -} diff --git a/test/client-side-encryption/spec/unified/QE-Text-suffixPreview.json b/test/client-side-encryption/spec/unified/QE-Text-suffixPreview.json deleted file mode 100644 index deec5e63b0..0000000000 --- a/test/client-side-encryption/spec/unified/QE-Text-suffixPreview.json +++ /dev/null @@ -1,338 +0,0 @@ -{ - "description": "QE-Text-suffixPreview", - "schemaVersion": "1.25", - "runOnRequirements": [ - { - "minServerVersion": "8.2.0", - "topologies": [ - "replicaset", - "sharded", - "load-balanced" - ], - "csfle": { - "minLibmongocryptVersion": "1.15.0" - } - } - ], - "createEntities": [ - { - "client": { - "id": "client", - "autoEncryptOpts": { - "keyVaultNamespace": "keyvault.datakeys", - "kmsProviders": { - "local": { - "key": "Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk" - } - } - }, - "observeEvents": [ - "commandStartedEvent" - ] - } - }, - { - "database": { - "id": "db", - "client": "client", - "databaseName": "db" - } - }, - { - "collection": { - "id": "coll", - "database": "db", - "collectionName": "coll" - } - } - ], - "initialData": [ - { - "databaseName": "keyvault", - "collectionName": "datakeys", - "documents": [ - { - "_id": { - "$binary": { - "base64": "q83vqxI0mHYSNBI0VniQEg==", - "subType": "04" - } - }, - "keyMaterial": { - "$binary": { - "base64": "HBk9BWihXExNDvTp1lUxOuxuZK2Pe2ZdVdlsxPEBkiO1bS4mG5NNDsQ7zVxJAH8BtdOYp72Ku4Y3nwc0BUpIKsvAKX4eYXtlhv5zUQxWdeNFhg9qK7qb8nqhnnLeT0f25jFSqzWJoT379hfwDeu0bebJHr35QrJ8myZdPMTEDYF08QYQ48ShRBli0S+QzBHHAQiM2iJNr4svg2WR8JSeWQ==", - "subType": "00" - } - }, - "creationDate": { - "$date": { - "$numberLong": "1648914851981" - } - }, - "updateDate": { - "$date": { - "$numberLong": "1648914851981" - } - }, - "status": { - "$numberInt": "0" - }, - "masterKey": { - "provider": "local" - } - } - ] - }, - { - "databaseName": "db", - "collectionName": "coll", - "documents": [], - "createOptions": { - "encryptedFields": { - "fields": [ - { - "keyId": { - "$binary": { - "base64": "q83vqxI0mHYSNBI0VniQEg==", - "subType": "04" - } - }, - "path": "encryptedText", - "bsonType": "string", - "queries": [ - { - "queryType": "suffixPreview", - "contention": { - "$numberLong": "0" - }, - "strMinQueryLength": { - "$numberLong": "3" - }, - "strMaxQueryLength": { - "$numberLong": "30" - }, - "caseSensitive": true, - "diacriticSensitive": true - } - ] - } - ] - } - } - } - ], - "tests": [ - { - "description": "Insert QE suffixPreview", - "operations": [ - { - "name": "insertOne", - "arguments": { - "document": { - "_id": 1, - "encryptedText": "foobar" - } - }, - "object": "coll" - } - ], - "expectEvents": [ - { - "client": "client", - "events": [ - { - "commandStartedEvent": { - "command": { - "listCollections": 1, - "filter": { - "name": "coll" - } - }, - "commandName": "listCollections" - } - }, - { - "commandStartedEvent": { - "command": { - "find": "datakeys", - "filter": { - "$or": [ - { - "_id": { - "$in": [ - { - "$binary": { - "base64": "q83vqxI0mHYSNBI0VniQEg==", - "subType": "04" - } - } - ] - } - }, - { - "keyAltNames": { - "$in": [] - } - } - ] - }, - "$db": "keyvault", - "readConcern": { - "level": "majority" - } - }, - "commandName": "find" - } - }, - { - "commandStartedEvent": { - "command": { - "insert": "coll", - "documents": [ - { - "_id": 1, - "encryptedText": { - "$$type": "binData" - } - } - ], - "ordered": true - }, - "commandName": "insert" - } - } - ] - } - ] - }, - { - "description": "Query with matching $encStrStartsWith", - "operations": [ - { - "name": "insertOne", - "arguments": { - "document": { - "_id": 1, - "encryptedText": "foobar" - } - }, - "object": "coll" - }, - { - "name": "find", - "arguments": { - "filter": { - "$expr": { - "$encStrEndsWith": { - "input": "$encryptedText", - "suffix": "bar" - } - } - } - }, - "object": "coll", - "expectResult": [ - { - "_id": { - "$numberInt": "1" - }, - "encryptedText": "foobar", - "__safeContent__": [ - { - "$binary": { - "base64": "wpaMBVDjL4bHf9EtSP52PJFzyNn1R19+iNI/hWtvzdk=", - "subType": "00" - } - }, - { - "$binary": { - "base64": "uDCWsucUsJemUP7pmeb+Kd8B9qupVzI8wnLFqX1rkiU=", - "subType": "00" - } - }, - { - "$binary": { - "base64": "W3E1x4bHZ8SEHFz4zwXM0G5Z5WSwBhnxE8x5/qdP6JM=", - "subType": "00" - } - }, - { - "$binary": { - "base64": "6g/TXVDDf6z+ntResIvTKWdmIy4ajQ1rhwdNZIiEG7A=", - "subType": "00" - } - }, - { - "$binary": { - "base64": "hU+u/T3D6dHDpT3d/v5AlgtRoAufCXCAyO2jQlgsnCw=", - "subType": "00" - } - }, - { - "$binary": { - "base64": "vrPnq0AtBIURNgNGA6HJL+5/p5SBWe+qz8505TRo/dE=", - "subType": "00" - } - }, - { - "$binary": { - "base64": "W5pylBxdv2soY2NcBfPiHDVLTS6tx+0ULkI8gysBeFY=", - "subType": "00" - } - }, - { - "$binary": { - "base64": "oWO3xX3x0bYUJGK2S1aPAmlU3Xtfsgb9lTZ6flGAlsg=", - "subType": "00" - } - }, - { - "$binary": { - "base64": "SjZGucTEUbdpd86O8yj1pyMyBOOKxvAQ9C8ngZ9C5UE=", - "subType": "00" - } - }, - { - "$binary": { - "base64": "CEaMZkxVDVbnXr+To0DOyvsva04UQkIYP3KtgYVVwf8=", - "subType": "00" - } - } - ] - } - ] - } - ] - }, - { - "description": "Query with non-matching $encStrEndsWith", - "operations": [ - { - "name": "insertOne", - "arguments": { - "document": { - "_id": 1, - "encryptedText": "foobar" - } - }, - "object": "coll" - }, - { - "name": "find", - "arguments": { - "filter": { - "$expr": { - "$encStrEndsWith": { - "input": "$encryptedText", - "suffix": "foo" - } - } - } - }, - "object": "coll", - "expectResult": [] - } - ] - } - ] -} diff --git a/test/client-side-encryption/spec/unified/fle2v2-BypassQueryAnalysis.json b/test/client-side-encryption/spec/unified/fle2v2-BypassQueryAnalysis.json deleted file mode 100644 index 0817508f8f..0000000000 --- a/test/client-side-encryption/spec/unified/fle2v2-BypassQueryAnalysis.json +++ /dev/null @@ -1,322 +0,0 @@ -{ - "description": "fle2v2-BypassQueryAnalysis", - "schemaVersion": "1.23", - "runOnRequirements": [ - { - "minServerVersion": "7.0.0", - "serverless": "forbid", - "csfle": true, - "topologies": [ - "replicaset", - "sharded", - "load-balanced" - ] - } - ], - "createEntities": [ - { - "client": { - "id": "client0", - "autoEncryptOpts": { - "kmsProviders": { - "local": { - "key": "Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk" - } - }, - "keyVaultNamespace": "keyvault.datakeys", - "bypassQueryAnalysis": true - }, - "observeEvents": [ - "commandStartedEvent" - ] - } - }, - { - "database": { - "id": "encryptedDB", - "client": "client0", - "databaseName": "default" - } - }, - { - "collection": { - "id": "encryptedColl", - "database": "encryptedDB", - "collectionName": "default" - } - }, - { - "client": { - "id": "client1" - } - }, - { - "database": { - "id": "unencryptedDB", - "client": "client1", - "databaseName": "default" - } - }, - { - "collection": { - "id": "unencryptedColl", - "database": "unencryptedDB", - "collectionName": "default" - } - } - ], - "initialData": [ - { - "databaseName": "keyvault", - "collectionName": "datakeys", - "documents": [ - { - "_id": { - "$binary": { - "base64": "EjRWeBI0mHYSNBI0VniQEg==", - "subType": "04" - } - }, - "keyMaterial": { - "$binary": { - "base64": "sHe0kz57YW7v8g9VP9sf/+K1ex4JqKc5rf/URX3n3p8XdZ6+15uXPaSayC6adWbNxkFskuMCOifDoTT+rkqMtFkDclOy884RuGGtUysq3X7zkAWYTKi8QAfKkajvVbZl2y23UqgVasdQu3OVBQCrH/xY00nNAs/52e958nVjBuzQkSb1T8pKJAyjZsHJ60+FtnfafDZSTAIBJYn7UWBCwQ==", - "subType": "00" - } - }, - "creationDate": { - "$date": { - "$numberLong": "1648914851981" - } - }, - "updateDate": { - "$date": { - "$numberLong": "1648914851981" - } - }, - "status": { - "$numberInt": "0" - }, - "masterKey": { - "provider": "local" - } - } - ] - }, - { - "databaseName": "default", - "collectionName": "default", - "documents": [], - "createOptions": { - "encryptedFields": { - "fields": [ - { - "keyId": { - "$binary": { - "base64": "EjRWeBI0mHYSNBI0VniQEg==", - "subType": "04" - } - }, - "path": "encryptedIndexed", - "bsonType": "string", - "queries": { - "queryType": "equality", - "contention": { - "$numberLong": "0" - } - } - }, - { - "keyId": { - "$binary": { - "base64": "q83vqxI0mHYSNBI0VniQEg==", - "subType": "04" - } - }, - "path": "encryptedUnindexed", - "bsonType": "string" - } - ] - } - } - } - ], - "tests": [ - { - "description": "BypassQueryAnalysis decrypts", - "operations": [ - { - "object": "encryptedColl", - "name": "insertOne", - "arguments": { - "document": { - "_id": 1, - "encryptedIndexed": { - "$binary": { - "base64": "C18BAAAFZAAgAAAAANnt+eLTkv4GdDPl8IAfJOvTzArOgFJQ2S/DcLza4W0DBXMAIAAAAAD2u+omZme3P2gBPehMQyQHQ153tPN1+z7bksYA9jKTpAVwADAAAAAAUnCOQqIvmR65YKyYnsiVfVrg9hwUVO3RhhKExo3RWOzgaS0QdsBL5xKFS0JhZSoWBXUAEAAAAAQSNFZ4EjSYdhI0EjRWeJASEHQAAgAAAAV2AFAAAAAAEjRWeBI0mHYSNBI0VniQEpQbp/ZJpWBKeDtKLiXb0P2E9wvc0g3f373jnYQYlJquOrlPOoEy3ngsHPJuSUijvWDsrQzqYa349K7G/66qaXEFZQAgAAAAAOuac/eRLYakKX6B0vZ1r3QodOQFfjqJD+xlGiPu4/PsBWwAIAAAAACkm0o9bj6j0HuADKc0svbqO2UHj6GrlNdF6yKNxh63xRJrAAAAAAAAAAAAAA==", - "subType": "06" - } - } - } - } - }, - { - "object": "encryptedColl", - "name": "find", - "arguments": { - "filter": { - "_id": 1 - } - }, - "expectResult": [ - { - "_id": 1, - "encryptedIndexed": "123" - } - ] - }, - { - "object": "unencryptedColl", - "name": "find", - "arguments": { - "filter": {} - }, - "expectResult": [ - { - "_id": 1, - "encryptedIndexed": { - "$$type": "binData" - }, - "__safeContent__": [ - { - "$binary": { - "base64": "31eCYlbQoVboc5zwC8IoyJVSkag9PxREka8dkmbXJeY=", - "subType": "00" - } - } - ] - } - ] - } - ], - "expectEvents": [ - { - "client": "client0", - "events": [ - { - "commandStartedEvent": { - "command": { - "listCollections": 1, - "filter": { - "name": "default" - } - }, - "commandName": "listCollections" - } - }, - { - "commandStartedEvent": { - "command": { - "insert": "default", - "documents": [ - { - "_id": 1, - "encryptedIndexed": { - "$binary": { - "base64": "C18BAAAFZAAgAAAAANnt+eLTkv4GdDPl8IAfJOvTzArOgFJQ2S/DcLza4W0DBXMAIAAAAAD2u+omZme3P2gBPehMQyQHQ153tPN1+z7bksYA9jKTpAVwADAAAAAAUnCOQqIvmR65YKyYnsiVfVrg9hwUVO3RhhKExo3RWOzgaS0QdsBL5xKFS0JhZSoWBXUAEAAAAAQSNFZ4EjSYdhI0EjRWeJASEHQAAgAAAAV2AFAAAAAAEjRWeBI0mHYSNBI0VniQEpQbp/ZJpWBKeDtKLiXb0P2E9wvc0g3f373jnYQYlJquOrlPOoEy3ngsHPJuSUijvWDsrQzqYa349K7G/66qaXEFZQAgAAAAAOuac/eRLYakKX6B0vZ1r3QodOQFfjqJD+xlGiPu4/PsBWwAIAAAAACkm0o9bj6j0HuADKc0svbqO2UHj6GrlNdF6yKNxh63xRJrAAAAAAAAAAAAAA==", - "subType": "06" - } - } - } - ], - "ordered": true, - "encryptionInformation": { - "type": 1, - "schema": { - "default.default": { - "escCollection": "enxcol_.default.esc", - "ecocCollection": "enxcol_.default.ecoc", - "fields": [ - { - "keyId": { - "$binary": { - "base64": "EjRWeBI0mHYSNBI0VniQEg==", - "subType": "04" - } - }, - "path": "encryptedIndexed", - "bsonType": "string", - "queries": { - "queryType": "equality", - "contention": { - "$numberLong": "0" - } - } - }, - { - "keyId": { - "$binary": { - "base64": "q83vqxI0mHYSNBI0VniQEg==", - "subType": "04" - } - }, - "path": "encryptedUnindexed", - "bsonType": "string" - } - ] - } - } - } - }, - "commandName": "insert" - } - }, - { - "commandStartedEvent": { - "command": { - "find": "default", - "filter": { - "_id": 1 - } - }, - "commandName": "find" - } - }, - { - "commandStartedEvent": { - "command": { - "find": "datakeys", - "filter": { - "$or": [ - { - "_id": { - "$in": [ - { - "$binary": { - "base64": "EjRWeBI0mHYSNBI0VniQEg==", - "subType": "04" - } - } - ] - } - }, - { - "keyAltNames": { - "$in": [] - } - } - ] - }, - "$db": "keyvault", - "readConcern": { - "level": "majority" - } - }, - "commandName": "find" - } - } - ] - } - ] - } - ] -} diff --git a/test/client-side-encryption/spec/unified/fle2v2-EncryptedFields-vs-EncryptedFieldsMap.json b/test/client-side-encryption/spec/unified/fle2v2-EncryptedFields-vs-EncryptedFieldsMap.json deleted file mode 100644 index b5f848c080..0000000000 --- a/test/client-side-encryption/spec/unified/fle2v2-EncryptedFields-vs-EncryptedFieldsMap.json +++ /dev/null @@ -1,256 +0,0 @@ -{ - "description": "fle2v2-EncryptedFields-vs-EncryptedFieldsMap", - "schemaVersion": "1.23", - "runOnRequirements": [ - { - "minServerVersion": "7.0.0", - "serverless": "forbid", - "csfle": true, - "topologies": [ - "replicaset", - "sharded", - "load-balanced" - ] - } - ], - "createEntities": [ - { - "client": { - "id": "client0", - "autoEncryptOpts": { - "kmsProviders": { - "local": { - "key": "Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk" - } - }, - "keyVaultNamespace": "keyvault.datakeys", - "encryptedFieldsMap": { - "default.default": { - "fields": [] - } - } - }, - "observeEvents": [ - "commandStartedEvent" - ] - } - }, - { - "database": { - "id": "encryptedDB", - "client": "client0", - "databaseName": "default" - } - }, - { - "collection": { - "id": "encryptedColl", - "database": "encryptedDB", - "collectionName": "default" - } - } - ], - "initialData": [ - { - "databaseName": "keyvault", - "collectionName": "datakeys", - "documents": [ - { - "_id": { - "$binary": { - "base64": "q83vqxI0mHYSNBI0VniQEg==", - "subType": "04" - } - }, - "keyMaterial": { - "$binary": { - "base64": "HBk9BWihXExNDvTp1lUxOuxuZK2Pe2ZdVdlsxPEBkiO1bS4mG5NNDsQ7zVxJAH8BtdOYp72Ku4Y3nwc0BUpIKsvAKX4eYXtlhv5zUQxWdeNFhg9qK7qb8nqhnnLeT0f25jFSqzWJoT379hfwDeu0bebJHr35QrJ8myZdPMTEDYF08QYQ48ShRBli0S+QzBHHAQiM2iJNr4svg2WR8JSeWQ==", - "subType": "00" - } - }, - "creationDate": { - "$date": { - "$numberLong": "1648914851981" - } - }, - "updateDate": { - "$date": { - "$numberLong": "1648914851981" - } - }, - "status": { - "$numberInt": "0" - }, - "masterKey": { - "provider": "local" - } - } - ] - }, - { - "databaseName": "default", - "collectionName": "default", - "documents": [], - "createOptions": { - "encryptedFields": { - "fields": [ - { - "keyId": { - "$binary": { - "base64": "EjRWeBI0mHYSNBI0VniQEg==", - "subType": "04" - } - }, - "path": "encryptedIndexed", - "bsonType": "string", - "queries": { - "queryType": "equality", - "contention": { - "$numberLong": "0" - } - } - }, - { - "keyId": { - "$binary": { - "base64": "q83vqxI0mHYSNBI0VniQEg==", - "subType": "04" - } - }, - "path": "encryptedUnindexed", - "bsonType": "string" - } - ] - } - } - } - ], - "tests": [ - { - "description": "encryptedFieldsMap is preferred over remote encryptedFields", - "operations": [ - { - "object": "encryptedColl", - "name": "insertOne", - "arguments": { - "document": { - "_id": 1, - "encryptedUnindexed": { - "$binary": { - "base64": "BqvN76sSNJh2EjQSNFZ4kBICTQaVZPWgXp41I7mPV1rLFTtw1tXzjcdSEyxpKKqujlko5TeizkB9hHQ009dVY1+fgIiDcefh+eQrm3CkhQ==", - "subType": "06" - } - } - } - } - }, - { - "object": "encryptedColl", - "name": "find", - "arguments": { - "filter": { - "_id": 1 - } - }, - "expectResult": [ - { - "_id": 1, - "encryptedUnindexed": "value123" - } - ] - } - ], - "expectEvents": [ - { - "client": "client0", - "events": [ - { - "commandStartedEvent": { - "databaseName": "default", - "commandName": "insert", - "command": { - "insert": "default", - "documents": [ - { - "_id": 1, - "encryptedUnindexed": { - "$binary": { - "base64": "BqvN76sSNJh2EjQSNFZ4kBICTQaVZPWgXp41I7mPV1rLFTtw1tXzjcdSEyxpKKqujlko5TeizkB9hHQ009dVY1+fgIiDcefh+eQrm3CkhQ==", - "subType": "06" - } - } - } - ], - "ordered": true - } - } - }, - { - "commandStartedEvent": { - "databaseName": "default", - "commandName": "find", - "command": { - "find": "default", - "filter": { - "_id": 1 - } - } - } - }, - { - "commandStartedEvent": { - "databaseName": "keyvault", - "commandName": "find", - "command": { - "find": "datakeys", - "filter": { - "$or": [ - { - "_id": { - "$in": [ - { - "$binary": { - "base64": "q83vqxI0mHYSNBI0VniQEg==", - "subType": "04" - } - } - ] - } - }, - { - "keyAltNames": { - "$in": [] - } - } - ] - }, - "$db": "keyvault", - "readConcern": { - "level": "majority" - } - } - } - } - ] - } - ], - "outcome": [ - { - "collectionName": "default", - "databaseName": "default", - "documents": [ - { - "_id": 1, - "encryptedUnindexed": { - "$binary": { - "base64": "BqvN76sSNJh2EjQSNFZ4kBICTQaVZPWgXp41I7mPV1rLFTtw1tXzjcdSEyxpKKqujlko5TeizkB9hHQ009dVY1+fgIiDcefh+eQrm3CkhQ==", - "subType": "06" - } - } - } - ] - } - ] - } - ] -} diff --git a/test/client-side-encryption/spec/unified/localSchema.json b/test/client-side-encryption/spec/unified/localSchema.json deleted file mode 100644 index aee323d949..0000000000 --- a/test/client-side-encryption/spec/unified/localSchema.json +++ /dev/null @@ -1,343 +0,0 @@ -{ - "description": "localSchema", - "schemaVersion": "1.23", - "runOnRequirements": [ - { - "minServerVersion": "4.1.10", - "csfle": true - } - ], - "createEntities": [ - { - "client": { - "id": "client0", - "autoEncryptOpts": { - "schemaMap": { - "default.default": { - "properties": { - "encrypted_w_altname": { - "encrypt": { - "keyId": "/altname", - "bsonType": "string", - "algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Random" - } - }, - "encrypted_string": { - "encrypt": { - "keyId": [ - { - "$binary": { - "base64": "AAAAAAAAAAAAAAAAAAAAAA==", - "subType": "04" - } - } - ], - "bsonType": "string", - "algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic" - } - }, - "random": { - "encrypt": { - "keyId": [ - { - "$binary": { - "base64": "AAAAAAAAAAAAAAAAAAAAAA==", - "subType": "04" - } - } - ], - "bsonType": "string", - "algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Random" - } - }, - "encrypted_string_equivalent": { - "encrypt": { - "keyId": [ - { - "$binary": { - "base64": "AAAAAAAAAAAAAAAAAAAAAA==", - "subType": "04" - } - } - ], - "bsonType": "string", - "algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic" - } - } - }, - "bsonType": "object" - } - }, - "keyVaultNamespace": "keyvault.datakeys", - "kmsProviders": { - "aws": { - "accessKeyId": { - "$$placeholder": 1 - }, - "secretAccessKey": { - "$$placeholder": 1 - }, - "sessionToken": { - "$$placeholder": 1 - } - } - } - }, - "observeEvents": [ - "commandStartedEvent" - ] - } - }, - { - "client": { - "id": "client1", - "autoEncryptOpts": { - "schemaMap": { - "default.default": { - "properties": { - "test": { - "bsonType": "string" - } - }, - "bsonType": "object", - "required": [ - "test" - ] - } - }, - "keyVaultNamespace": "keyvault.datakeys", - "kmsProviders": { - "aws": { - "accessKeyId": { - "$$placeholder": 1 - }, - "secretAccessKey": { - "$$placeholder": 1 - }, - "sessionToken": { - "$$placeholder": 1 - } - } - } - }, - "observeEvents": [ - "commandStartedEvent" - ] - } - }, - { - "database": { - "id": "encryptedDB", - "client": "client0", - "databaseName": "default" - } - }, - { - "collection": { - "id": "encryptedColl", - "database": "encryptedDB", - "collectionName": "default" - } - }, - { - "database": { - "id": "encryptedDB2", - "client": "client1", - "databaseName": "default" - } - }, - { - "collection": { - "id": "encryptedColl2", - "database": "encryptedDB2", - "collectionName": "default" - } - } - ], - "initialData": [ - { - "databaseName": "keyvault", - "collectionName": "datakeys", - "documents": [ - { - "status": 1, - "_id": { - "$binary": { - "base64": "AAAAAAAAAAAAAAAAAAAAAA==", - "subType": "04" - } - }, - "masterKey": { - "provider": "aws", - "key": "arn:aws:kms:us-east-1:579766882180:key/89fcc2c4-08b0-4bd9-9f25-e30687b580d0", - "region": "us-east-1" - }, - "updateDate": { - "$date": { - "$numberLong": "1552949630483" - } - }, - "keyMaterial": { - "$binary": { - "base64": "AQICAHhQNmWG2CzOm1dq3kWLM+iDUZhEqnhJwH9wZVpuZ94A8gEqnsxXlR51T5EbEVezUqqKAAAAwjCBvwYJKoZIhvcNAQcGoIGxMIGuAgEAMIGoBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDHa4jo6yp0Z18KgbUgIBEIB74sKxWtV8/YHje5lv5THTl0HIbhSwM6EqRlmBiFFatmEWaeMk4tO4xBX65eq670I5TWPSLMzpp8ncGHMmvHqRajNBnmFtbYxN3E3/WjxmdbOOe+OXpnGJPcGsftc7cB2shRfA4lICPnE26+oVNXT6p0Lo20nY5XC7jyCO", - "subType": "00" - } - }, - "creationDate": { - "$date": { - "$numberLong": "1552949630483" - } - }, - "keyAltNames": [ - "altname", - "another_altname" - ] - } - ] - }, - { - "databaseName": "default", - "collectionName": "default", - "documents": [] - } - ], - "tests": [ - { - "description": "A local schema should override", - "operations": [ - { - "object": "encryptedColl", - "name": "insertOne", - "arguments": { - "document": { - "_id": 1, - "encrypted_string": "string0" - } - } - }, - { - "object": "encryptedColl", - "name": "find", - "arguments": { - "filter": { - "_id": 1 - } - }, - "expectResult": [ - { - "_id": 1, - "encrypted_string": "string0" - } - ] - } - ], - "expectEvents": [ - { - "client": "client0", - "events": [ - { - "commandStartedEvent": { - "databaseName": "keyvault", - "commandName": "find", - "command": { - "find": "datakeys", - "filter": { - "$or": [ - { - "_id": { - "$in": [ - { - "$binary": { - "base64": "AAAAAAAAAAAAAAAAAAAAAA==", - "subType": "04" - } - } - ] - } - }, - { - "keyAltNames": { - "$in": [] - } - } - ] - }, - "readConcern": { - "level": "majority" - } - } - } - }, - { - "commandStartedEvent": { - "commandName": "insert", - "command": { - "insert": "default", - "documents": [ - { - "_id": 1, - "encrypted_string": { - "$binary": { - "base64": "AQAAAAAAAAAAAAAAAAAAAAACwj+3zkv2VM+aTfk60RqhXq6a/77WlLwu/BxXFkL7EppGsju/m8f0x5kBDD3EZTtGALGXlym5jnpZAoSIkswHoA==", - "subType": "06" - } - } - } - ], - "ordered": true - } - } - }, - { - "commandStartedEvent": { - "commandName": "find", - "command": { - "find": "default", - "filter": { - "_id": 1 - } - } - } - } - ] - } - ], - "outcome": [ - { - "collectionName": "default", - "databaseName": "default", - "documents": [ - { - "_id": 1, - "encrypted_string": { - "$binary": { - "base64": "AQAAAAAAAAAAAAAAAAAAAAACwj+3zkv2VM+aTfk60RqhXq6a/77WlLwu/BxXFkL7EppGsju/m8f0x5kBDD3EZTtGALGXlym5jnpZAoSIkswHoA==", - "subType": "06" - } - } - } - ] - } - ] - }, - { - "description": "A local schema with no encryption is an error", - "operations": [ - { - "object": "encryptedColl2", - "name": "insertOne", - "arguments": { - "document": { - "_id": 1, - "encrypted_string": "string0" - } - }, - "expectError": { - "isError": true, - "errorContains": "JSON schema keyword 'required' is only allowed with a remote schema" - } - } - ] - } - ] -} diff --git a/test/client-side-encryption/spec/unified/maxWireVersion.json b/test/client-side-encryption/spec/unified/maxWireVersion.json deleted file mode 100644 index d0af75ac99..0000000000 --- a/test/client-side-encryption/spec/unified/maxWireVersion.json +++ /dev/null @@ -1,101 +0,0 @@ -{ - "description": "maxWireVersion", - "schemaVersion": "1.23", - "runOnRequirements": [ - { - "maxServerVersion": "4.0.99", - "csfle": true - } - ], - "createEntities": [ - { - "client": { - "id": "client0", - "autoEncryptOpts": { - "kmsProviders": { - "aws": {} - }, - "keyVaultNamespace": "keyvault.datakeys", - "extraOptions": { - "mongocryptdBypassSpawn": true - } - } - } - }, - { - "database": { - "id": "database0", - "client": "client0", - "databaseName": "default" - } - }, - { - "collection": { - "id": "collection0", - "database": "database0", - "collectionName": "default" - } - } - ], - "initialData": [ - { - "databaseName": "keyvault", - "collectionName": "datakeys", - "documents": [ - { - "status": 1, - "_id": { - "$binary": { - "base64": "AAAAAAAAAAAAAAAAAAAAAA==", - "subType": "04" - } - }, - "masterKey": { - "provider": "aws", - "key": "arn:aws:kms:us-east-1:579766882180:key/89fcc2c4-08b0-4bd9-9f25-e30687b580d0", - "region": "us-east-1" - }, - "updateDate": { - "$date": { - "$numberLong": "1552949630483" - } - }, - "keyMaterial": { - "$binary": { - "base64": "AQICAHhQNmWG2CzOm1dq3kWLM+iDUZhEqnhJwH9wZVpuZ94A8gEqnsxXlR51T5EbEVezUqqKAAAAwjCBvwYJKoZIhvcNAQcGoIGxMIGuAgEAMIGoBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDHa4jo6yp0Z18KgbUgIBEIB74sKxWtV8/YHje5lv5THTl0HIbhSwM6EqRlmBiFFatmEWaeMk4tO4xBX65eq670I5TWPSLMzpp8ncGHMmvHqRajNBnmFtbYxN3E3/WjxmdbOOe+OXpnGJPcGsftc7cB2shRfA4lICPnE26+oVNXT6p0Lo20nY5XC7jyCO", - "subType": "00" - } - }, - "creationDate": { - "$date": { - "$numberLong": "1552949630483" - } - }, - "keyAltNames": [ - "altname", - "another_altname" - ] - } - ] - } - ], - "tests": [ - { - "description": "operation fails with maxWireVersion < 8", - "operations": [ - { - "name": "insertOne", - "object": "collection0", - "arguments": { - "document": { - "encrypted_string": "string0" - } - }, - "expectError": { - "errorContains": "Auto-encryption requires a minimum MongoDB version of 4.2" - } - } - ] - } - ] -} diff --git a/test/csot/deprecated-options.json b/test/csot/deprecated-options.json index 647e1bf792..d3e4631ff4 100644 --- a/test/csot/deprecated-options.json +++ b/test/csot/deprecated-options.json @@ -6750,23 +6750,16 @@ } } }, - { - "name": "createIndex", - "object": "collection", - "arguments": { - "keys": { - "x": 1 - }, - "timeoutMS": 100000, - "name": "x_1" - } - }, { "name": "dropIndex", "object": "collection", "arguments": { "timeoutMS": 100000, "name": "x_1" + }, + "expectError": { + "isClientError": false, + "isTimeoutError": false } } ] @@ -6822,23 +6815,16 @@ ] } }, - { - "name": "createIndex", - "object": "collection", - "arguments": { - "keys": { - "x": 1 - }, - "timeoutMS": 100000, - "name": "x_1" - } - }, { "name": "dropIndex", "object": "collection", "arguments": { "timeoutMS": 100000, "name": "x_1" + }, + "expectError": { + "isClientError": false, + "isTimeoutError": false } } ], @@ -6846,12 +6832,6 @@ { "client": "client", "events": [ - { - "commandStartedEvent": { - "commandName": "createIndexes", - "databaseName": "test" - } - }, { "commandStartedEvent": { "commandName": "dropIndexes", @@ -6923,16 +6903,6 @@ ] } }, - { - "name": "createIndex", - "object": "collection", - "arguments": { - "keys": { - "x": 1 - }, - "name": "x_1" - } - }, { "name": "dropIndex", "object": "collection", @@ -6940,6 +6910,10 @@ "timeoutMS": 1000, "maxTimeMS": 5000, "name": "x_1" + }, + "expectError": { + "isClientError": false, + "isTimeoutError": false } } ], @@ -6947,12 +6921,6 @@ { "client": "client", "events": [ - { - "commandStartedEvent": { - "commandName": "createIndexes", - "databaseName": "test" - } - }, { "commandStartedEvent": { "commandName": "dropIndexes", @@ -7035,17 +7003,6 @@ } } }, - { - "name": "createIndex", - "object": "collection", - "arguments": { - "keys": { - "x": 1 - }, - "name": "x_1", - "timeoutMS": 100000 - } - }, { "name": "dropIndexes", "object": "collection", diff --git a/test/csot/global-timeoutMS.json b/test/csot/global-timeoutMS.json index f1edbe68e3..740bbad2e2 100644 --- a/test/csot/global-timeoutMS.json +++ b/test/csot/global-timeoutMS.json @@ -5621,21 +5621,15 @@ } } }, - { - "name": "createIndex", - "object": "collection", - "arguments": { - "keys": { - "x": 1 - }, - "name": "x_1" - } - }, { "name": "dropIndex", "object": "collection", "arguments": { "name": "x_1" + }, + "expectError": { + "isClientError": false, + "isTimeoutError": false } } ], @@ -5643,12 +5637,6 @@ { "client": "client", "events": [ - { - "commandStartedEvent": { - "commandName": "createIndexes", - "databaseName": "test" - } - }, { "commandStartedEvent": { "commandName": "dropIndexes", diff --git a/test/csot/override-operation-timeoutMS.json b/test/csot/override-operation-timeoutMS.json index f33f876137..6fa0bd802a 100644 --- a/test/csot/override-operation-timeoutMS.json +++ b/test/csot/override-operation-timeoutMS.json @@ -3378,23 +3378,15 @@ } } }, - { - "name": "createIndex", - "object": "collection", - "arguments": { - "keys": { - "x": 1 - }, - "timeoutMS": 1000, - "name": "x_1" - } - }, { "name": "dropIndex", "object": "collection", "arguments": { "timeoutMS": 1000, "name": "x_1" + }, + "expectError": { + "isTimeoutError": false } } ], @@ -3402,12 +3394,6 @@ { "client": "client", "events": [ - { - "commandStartedEvent": { - "commandName": "createIndexes", - "databaseName": "test" - } - }, { "commandStartedEvent": { "commandName": "dropIndexes", @@ -3450,23 +3436,15 @@ } } }, - { - "name": "createIndex", - "object": "collection", - "arguments": { - "keys": { - "x": 1 - }, - "timeoutMS": 0, - "name": "x_1" - } - }, { "name": "dropIndex", "object": "collection", "arguments": { "timeoutMS": 0, "name": "x_1" + }, + "expectError": { + "isTimeoutError": false } } ], @@ -3474,12 +3452,6 @@ { "client": "client", "events": [ - { - "commandStartedEvent": { - "commandName": "createIndexes", - "databaseName": "test" - } - }, { "commandStartedEvent": { "commandName": "dropIndexes", diff --git a/test/csot/tailable-awaitData.json b/test/csot/tailable-awaitData.json index 80e95ca906..81683d3993 100644 --- a/test/csot/tailable-awaitData.json +++ b/test/csot/tailable-awaitData.json @@ -78,7 +78,7 @@ ] }, { - "description": "error on find if maxAwaitTimeMS is greater than timeoutMS", + "description": "error if maxAwaitTimeMS is greater than timeoutMS", "operations": [ { "name": "find", @@ -90,50 +90,13 @@ "maxAwaitTimeMS": 10 }, "expectError": { - "isClientError": true, - "isTimeoutError": false - } - } - ] - }, - { - "description": "error on aggregate if maxAwaitTimeMS is greater than timeoutMS", - "operations": [ - { - "name": "aggregate", - "object": "collection", - "arguments": { - "pipeline": [], - "timeoutMS": 5, - "maxAwaitTimeMS": 10 - }, - "expectError": { - "isClientError": true, - "isTimeoutError": false - } - } - ] - }, - { - "description": "error on watch if maxAwaitTimeMS is greater than timeoutMS", - "operations": [ - { - "name": "createChangeStream", - "object": "collection", - "arguments": { - "pipeline": [], - "timeoutMS": 5, - "maxAwaitTimeMS": 10 - }, - "expectError": { - "isClientError": true, - "isTimeoutError": false + "isClientError": true } } ] }, { - "description": "error on find if maxAwaitTimeMS is equal to timeoutMS", + "description": "error if maxAwaitTimeMS is equal to timeoutMS", "operations": [ { "name": "find", @@ -145,44 +108,7 @@ "maxAwaitTimeMS": 5 }, "expectError": { - "isClientError": true, - "isTimeoutError": false - } - } - ] - }, - { - "description": "error on aggregate if maxAwaitTimeMS is equal to timeoutMS", - "operations": [ - { - "name": "aggregate", - "object": "collection", - "arguments": { - "pipeline": [], - "timeoutMS": 5, - "maxAwaitTimeMS": 5 - }, - "expectError": { - "isClientError": true, - "isTimeoutError": false - } - } - ] - }, - { - "description": "error on watch if maxAwaitTimeMS is equal to timeoutMS", - "operations": [ - { - "name": "createChangeStream", - "object": "collection", - "arguments": { - "pipeline": [], - "timeoutMS": 5, - "maxAwaitTimeMS": 5 - }, - "expectError": { - "isClientError": true, - "isTimeoutError": false + "isClientError": true } } ] diff --git a/test/helpers.py b/test/helpers.py index 163bf01c12..22bdc0d25d 100644 --- a/test/helpers.py +++ b/test/helpers.py @@ -12,22 +12,137 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Shared helper methods for pymongo, bson, and gridfs test suites.""" +"""Shared constants and helper methods for pymongo, bson, and gridfs test suites.""" from __future__ import annotations import asyncio +import base64 +import gc +import multiprocessing +import os +import signal +import socket +import subprocess +import sys import threading +import time import traceback -from functools import wraps -from typing import Optional, no_type_check +import unittest +import warnings +from inspect import iscoroutinefunction -from bson import SON -from pymongo import common from pymongo._asyncio_task import create_task + +try: + import ipaddress + + HAVE_IPADDRESS = True +except ImportError: + HAVE_IPADDRESS = False +from functools import wraps +from typing import Any, Callable, Dict, Generator, Optional, no_type_check +from unittest import SkipTest + +from bson.son import SON +from pymongo import common, message from pymongo.read_preferences import ReadPreference +from pymongo.ssl_support import HAVE_SSL, _ssl # type:ignore[attr-defined] +from pymongo.synchronous.uri_parser import parse_uri + +if HAVE_SSL: + import ssl _IS_SYNC = True +# Enable debug output for uncollectable objects. PyPy does not have set_debug. +if hasattr(gc, "set_debug"): + gc.set_debug( + gc.DEBUG_UNCOLLECTABLE | getattr(gc, "DEBUG_OBJECTS", 0) | getattr(gc, "DEBUG_INSTANCES", 0) + ) + +# The host and port of a single mongod or mongos, or the seed host +# for a replica set. +host = os.environ.get("DB_IP", "localhost") +port = int(os.environ.get("DB_PORT", 27017)) +IS_SRV = "mongodb+srv" in host + +db_user = os.environ.get("DB_USER", "user") +db_pwd = os.environ.get("DB_PASSWORD", "password") + +CERT_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), "certificates") +CLIENT_PEM = os.environ.get("CLIENT_PEM", os.path.join(CERT_PATH, "client.pem")) +CA_PEM = os.environ.get("CA_PEM", os.path.join(CERT_PATH, "ca.pem")) + +TLS_OPTIONS: Dict = {"tls": True} +if CLIENT_PEM: + TLS_OPTIONS["tlsCertificateKeyFile"] = CLIENT_PEM +if CA_PEM: + TLS_OPTIONS["tlsCAFile"] = CA_PEM + +COMPRESSORS = os.environ.get("COMPRESSORS") +MONGODB_API_VERSION = os.environ.get("MONGODB_API_VERSION") +TEST_LOADBALANCER = bool(os.environ.get("TEST_LOAD_BALANCER")) +SINGLE_MONGOS_LB_URI = os.environ.get("SINGLE_MONGOS_LB_URI") +MULTI_MONGOS_LB_URI = os.environ.get("MULTI_MONGOS_LB_URI") + +if TEST_LOADBALANCER: + res = parse_uri(SINGLE_MONGOS_LB_URI or "") + host, port = res["nodelist"][0] + db_user = res["username"] or db_user + db_pwd = res["password"] or db_pwd + + +# Shared KMS data. +LOCAL_MASTER_KEY = base64.b64decode( + b"Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ" + b"5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk" +) +AWS_CREDS = { + "accessKeyId": os.environ.get("FLE_AWS_KEY", ""), + "secretAccessKey": os.environ.get("FLE_AWS_SECRET", ""), +} +AWS_CREDS_2 = { + "accessKeyId": os.environ.get("FLE_AWS_KEY2", ""), + "secretAccessKey": os.environ.get("FLE_AWS_SECRET2", ""), +} +AZURE_CREDS = { + "tenantId": os.environ.get("FLE_AZURE_TENANTID", ""), + "clientId": os.environ.get("FLE_AZURE_CLIENTID", ""), + "clientSecret": os.environ.get("FLE_AZURE_CLIENTSECRET", ""), +} +GCP_CREDS = { + "email": os.environ.get("FLE_GCP_EMAIL", ""), + "privateKey": os.environ.get("FLE_GCP_PRIVATEKEY", ""), +} +KMIP_CREDS = {"endpoint": os.environ.get("FLE_KMIP_ENDPOINT", "localhost:5698")} + +# Ensure Evergreen metadata doesn't result in truncation +os.environ.setdefault("MONGOB_LOG_MAX_DOCUMENT_LENGTH", "2000") + + +def is_server_resolvable(): + """Returns True if 'server' is resolvable.""" + socket_timeout = socket.getdefaulttimeout() + socket.setdefaulttimeout(1) + try: + try: + socket.gethostbyname("server") + return True + except OSError: + return False + finally: + socket.setdefaulttimeout(socket_timeout) + + +def _create_user(authdb, user, pwd=None, roles=None, **kwargs): + cmd = SON([("createUser", user)]) + # X509 doesn't use a password + if pwd: + cmd["pwd"] = pwd + cmd["roles"] = roles or ["root"] + cmd.update(**kwargs) + return authdb.command(cmd) + def repl_set_step_down(client, **kwargs): """Run replSetStepDown, first unfreezing a secondary with replSetFreeze.""" @@ -122,10 +237,133 @@ def __del__(self): raise Exception(msg) +def _all_users(db): + return {u["user"] for u in db.command("usersInfo").get("users", [])} + + +def sanitize_cmd(cmd): + cp = cmd.copy() + cp.pop("$clusterTime", None) + cp.pop("$db", None) + cp.pop("$readPreference", None) + cp.pop("lsid", None) + if MONGODB_API_VERSION: + # Stable API parameters + cp.pop("apiVersion", None) + # OP_MSG encoding may move the payload type one field to the + # end of the command. Do the same here. + name = next(iter(cp)) + try: + identifier = message._FIELD_MAP[name] + docs = cp.pop(identifier) + cp[identifier] = docs + except KeyError: + pass + return cp + + +def sanitize_reply(reply): + cp = reply.copy() + cp.pop("$clusterTime", None) + cp.pop("operationTime", None) + return cp + + +def print_thread_tracebacks() -> None: + """Print all Python thread tracebacks.""" + for thread_id, frame in sys._current_frames().items(): + sys.stderr.write(f"\n--- Traceback for thread {thread_id} ---\n") + traceback.print_stack(frame, file=sys.stderr) + + +def print_thread_stacks(pid: int) -> None: + """Print all C-level thread stacks for a given process id.""" + if sys.platform == "darwin": + cmd = ["lldb", "--attach-pid", f"{pid}", "--batch", "--one-line", '"thread backtrace all"'] + else: + cmd = ["gdb", f"--pid={pid}", "--batch", '--eval-command="thread apply all bt"'] + + try: + res = subprocess.run( + cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, encoding="utf-8" + ) + except Exception as exc: + sys.stderr.write(f"Could not print C-level thread stacks because {cmd[0]} failed: {exc}") + else: + sys.stderr.write(res.stdout) + + # Global knobs to speed up the test suite. global_knobs = client_knobs(events_queue_frequency=0.05) +def _get_executors(topology): + executors = [] + for server in topology._servers.values(): + # Some MockMonitor do not have an _executor. + if hasattr(server._monitor, "_executor"): + executors.append(server._monitor._executor) + if hasattr(server._monitor, "_rtt_monitor"): + executors.append(server._monitor._rtt_monitor._executor) + executors.append(topology._Topology__events_executor) + if topology._srv_monitor: + executors.append(topology._srv_monitor._executor) + + return [e for e in executors if e is not None] + + +def print_running_topology(topology): + running = [e for e in _get_executors(topology) if not e._stopped] + if running: + print( + "WARNING: found Topology with running threads:\n" + f" Threads: {running}\n" + f" Topology: {topology}\n" + f" Creation traceback:\n{topology._settings._stack}" + ) + + +def test_cases(suite): + """Iterator over all TestCases within a TestSuite.""" + for suite_or_case in suite._tests: + if isinstance(suite_or_case, unittest.TestCase): + # unittest.TestCase + yield suite_or_case + else: + # unittest.TestSuite + yield from test_cases(suite_or_case) + + +# Helper method to workaround https://bugs.python.org/issue21724 +def clear_warning_registry(): + """Clear the __warningregistry__ for all modules.""" + for _, module in list(sys.modules.items()): + if hasattr(module, "__warningregistry__"): + module.__warningregistry__ = {} # type:ignore[attr-defined] + + +class SystemCertsPatcher: + def __init__(self, ca_certs): + if ( + ssl.OPENSSL_VERSION.lower().startswith("libressl") + and sys.platform == "darwin" + and not _ssl.IS_PYOPENSSL + ): + raise SkipTest( + "LibreSSL on OSX doesn't support setting CA certificates " + "using SSL_CERT_FILE environment variable." + ) + self.original_certs = os.environ.get("SSL_CERT_FILE") + # Tell OpenSSL where CA certificates live. + os.environ["SSL_CERT_FILE"] = ca_certs + + def disable(self): + if self.original_certs is None: + os.environ.pop("SSL_CERT_FILE") + else: + os.environ["SSL_CERT_FILE"] = self.original_certs + + if _IS_SYNC: PARENT = threading.Thread else: diff --git a/test/helpers_shared.py b/test/helpers_shared.py deleted file mode 100644 index 49cf131808..0000000000 --- a/test/helpers_shared.py +++ /dev/null @@ -1,271 +0,0 @@ -# Copyright 2019-present MongoDB, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import annotations - -import base64 -import gc -import os -import socket -import subprocess -import sys -import traceback -import unittest -from pathlib import Path - -try: - import ipaddress - - HAVE_IPADDRESS = True -except ImportError: - HAVE_IPADDRESS = False -from functools import wraps -from typing import no_type_check -from unittest import SkipTest - -from bson.son import SON -from pymongo import message -from pymongo.ssl_support import HAVE_SSL, _ssl # type:ignore[attr-defined] -from pymongo.synchronous.uri_parser import parse_uri - -if HAVE_SSL: - import ssl - - -# Enable debug output for uncollectable objects. PyPy does not have set_debug. -if hasattr(gc, "set_debug"): - gc.set_debug( - gc.DEBUG_UNCOLLECTABLE | getattr(gc, "DEBUG_OBJECTS", 0) | getattr(gc, "DEBUG_INSTANCES", 0) - ) - -# The host and port of a single mongod or mongos, or the seed host -# for a replica set. -host = os.environ.get("DB_IP", "localhost") -port = int(os.environ.get("DB_PORT", 27017)) -IS_SRV = "mongodb+srv" in host - -db_user = os.environ.get("DB_USER", "user") -db_pwd = os.environ.get("DB_PASSWORD", "password") - -HERE = Path(__file__).absolute() -CERT_PATH = str(HERE.parent / "certificates") -CLIENT_PEM = os.environ.get("CLIENT_PEM", os.path.join(CERT_PATH, "client.pem")) -CA_PEM = os.environ.get("CA_PEM", os.path.join(CERT_PATH, "ca.pem")) - -TLS_OPTIONS: dict = {"tls": True} -if CLIENT_PEM: - TLS_OPTIONS["tlsCertificateKeyFile"] = CLIENT_PEM -if CA_PEM: - TLS_OPTIONS["tlsCAFile"] = CA_PEM - -COMPRESSORS = os.environ.get("COMPRESSORS") -MONGODB_API_VERSION = os.environ.get("MONGODB_API_VERSION") -TEST_LOADBALANCER = bool(os.environ.get("TEST_LOAD_BALANCER")) -SINGLE_MONGOS_LB_URI = os.environ.get("SINGLE_MONGOS_LB_URI") -MULTI_MONGOS_LB_URI = os.environ.get("MULTI_MONGOS_LB_URI") - -if TEST_LOADBALANCER: - res = parse_uri(SINGLE_MONGOS_LB_URI or "") - host, port = res["nodelist"][0] - db_user = res["username"] or db_user - db_pwd = res["password"] or db_pwd - - -# Shared KMS data. -LOCAL_MASTER_KEY = base64.b64decode( - b"Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ" - b"5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk" -) -AWS_CREDS = { - "accessKeyId": os.environ.get("FLE_AWS_KEY", ""), - "secretAccessKey": os.environ.get("FLE_AWS_SECRET", ""), -} -AWS_CREDS_2 = { - "accessKeyId": os.environ.get("FLE_AWS_KEY2", ""), - "secretAccessKey": os.environ.get("FLE_AWS_SECRET2", ""), -} -AZURE_CREDS = { - "tenantId": os.environ.get("FLE_AZURE_TENANTID", ""), - "clientId": os.environ.get("FLE_AZURE_CLIENTID", ""), - "clientSecret": os.environ.get("FLE_AZURE_CLIENTSECRET", ""), -} -GCP_CREDS = { - "email": os.environ.get("FLE_GCP_EMAIL", ""), - "privateKey": os.environ.get("FLE_GCP_PRIVATEKEY", ""), -} -KMIP_CREDS = {"endpoint": os.environ.get("FLE_KMIP_ENDPOINT", "localhost:5698")} -AWS_TEMP_CREDS = { - "accessKeyId": os.environ.get("CSFLE_AWS_TEMP_ACCESS_KEY_ID", ""), - "secretAccessKey": os.environ.get("CSFLE_AWS_TEMP_SECRET_ACCESS_KEY", ""), - "sessionToken": os.environ.get("CSFLE_AWS_TEMP_SESSION_TOKEN", ""), -} - -ALL_KMS_PROVIDERS = dict( - aws=AWS_CREDS, - azure=AZURE_CREDS, - gcp=GCP_CREDS, - local=dict(key=LOCAL_MASTER_KEY), - kmip=KMIP_CREDS, -) -DEFAULT_KMS_TLS = dict(kmip=dict(tlsCAFile=CA_PEM, tlsCertificateKeyFile=CLIENT_PEM)) - -# Ensure Evergreen metadata doesn't result in truncation -os.environ.setdefault("MONGOB_LOG_MAX_DOCUMENT_LENGTH", "2000") - - -def is_server_resolvable(): - """Returns True if 'server' is resolvable.""" - socket_timeout = socket.getdefaulttimeout() - socket.setdefaulttimeout(1) - try: - try: - socket.gethostbyname("server") - return True - except OSError: - return False - finally: - socket.setdefaulttimeout(socket_timeout) - - -def _create_user(authdb, user, pwd=None, roles=None, **kwargs): - cmd = SON([("createUser", user)]) - # X509 doesn't use a password - if pwd: - cmd["pwd"] = pwd - cmd["roles"] = roles or ["root"] - cmd.update(**kwargs) - return authdb.command(cmd) - - -def _all_users(db): - return {u["user"] for u in db.command("usersInfo").get("users", [])} - - -def sanitize_cmd(cmd): - cp = cmd.copy() - cp.pop("$clusterTime", None) - cp.pop("$db", None) - cp.pop("$readPreference", None) - cp.pop("lsid", None) - if MONGODB_API_VERSION: - # Stable API parameters - cp.pop("apiVersion", None) - # OP_MSG encoding may move the payload type one field to the - # end of the command. Do the same here. - name = next(iter(cp)) - try: - identifier = message._FIELD_MAP[name] - docs = cp.pop(identifier) - cp[identifier] = docs - except KeyError: - pass - return cp - - -def sanitize_reply(reply): - cp = reply.copy() - cp.pop("$clusterTime", None) - cp.pop("operationTime", None) - return cp - - -def print_thread_tracebacks() -> None: - """Print all Python thread tracebacks.""" - for thread_id, frame in sys._current_frames().items(): - sys.stderr.write(f"\n--- Traceback for thread {thread_id} ---\n") - traceback.print_stack(frame, file=sys.stderr) - - -def print_thread_stacks(pid: int) -> None: - """Print all C-level thread stacks for a given process id.""" - if sys.platform == "darwin": - cmd = ["lldb", "--attach-pid", f"{pid}", "--batch", "--one-line", '"thread backtrace all"'] - else: - cmd = ["gdb", f"--pid={pid}", "--batch", '--eval-command="thread apply all bt"'] - - try: - res = subprocess.run( - cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, encoding="utf-8" - ) - except Exception as exc: - sys.stderr.write(f"Could not print C-level thread stacks because {cmd[0]} failed: {exc}") - else: - sys.stderr.write(res.stdout) - - -def _get_executors(topology): - executors = [] - for server in topology._servers.values(): - # Some MockMonitor do not have an _executor. - if hasattr(server._monitor, "_executor"): - executors.append(server._monitor._executor) - if hasattr(server._monitor, "_rtt_monitor"): - executors.append(server._monitor._rtt_monitor._executor) - executors.append(topology._Topology__events_executor) - if topology._srv_monitor: - executors.append(topology._srv_monitor._executor) - - return [e for e in executors if e is not None] - - -def print_running_topology(topology): - running = [e for e in _get_executors(topology) if not e._stopped] - if running: - print( - "WARNING: found Topology with running threads:\n" - f" Threads: {running}\n" - f" Topology: {topology}\n" - f" Creation traceback:\n{topology._settings._stack}" - ) - - -def test_cases(suite): - """Iterator over all TestCases within a TestSuite.""" - for suite_or_case in suite._tests: - if isinstance(suite_or_case, unittest.TestCase): - # unittest.TestCase - yield suite_or_case - else: - # unittest.TestSuite - yield from test_cases(suite_or_case) - - -# Helper method to workaround https://bugs.python.org/issue21724 -def clear_warning_registry(): - """Clear the __warningregistry__ for all modules.""" - for _, module in list(sys.modules.items()): - if hasattr(module, "__warningregistry__"): - module.__warningregistry__ = {} # type:ignore[attr-defined] - - -class SystemCertsPatcher: - def __init__(self, ca_certs): - if ( - ssl.OPENSSL_VERSION.lower().startswith("libressl") - and sys.platform == "darwin" - and not _ssl.IS_PYOPENSSL - ): - raise SkipTest( - "LibreSSL on OSX doesn't support setting CA certificates " - "using SSL_CERT_FILE environment variable." - ) - self.original_certs = os.environ.get("SSL_CERT_FILE") - # Tell OpenSSL where CA certificates live. - os.environ["SSL_CERT_FILE"] = ca_certs - - def disable(self): - if self.original_certs is None: - os.environ.pop("SSL_CERT_FILE") - else: - os.environ["SSL_CERT_FILE"] = self.original_certs diff --git a/test/test_collection.py b/test/test_collection.py index b1947259ba..0dce88423b 100644 --- a/test/test_collection.py +++ b/test/test_collection.py @@ -1305,7 +1305,7 @@ def test_error_code(self): self.assertIn(exc.code, (9, 10147, 16840, 17009)) # Just check that we set the error document. Fields # vary by MongoDB version. - self.assertIsNotNone(exc.details) + self.assertTrue(exc.details is not None) else: self.fail("OperationFailure was not raised") diff --git a/test/test_custom_types.py b/test/test_custom_types.py index aba6b55119..7360f2b18b 100644 --- a/test/test_custom_types.py +++ b/test/test_custom_types.py @@ -23,7 +23,6 @@ from random import random from typing import Any, Tuple, Type, no_type_check -from bson.decimal128 import DecimalDecoder, DecimalEncoder from gridfs.synchronous.grid_file import GridIn, GridOut sys.path[0:0] = [""] @@ -60,7 +59,29 @@ _IS_SYNC = True -DECIMAL_CODECOPTS = CodecOptions(type_registry=TypeRegistry([DecimalEncoder(), DecimalDecoder()])) +class DecimalEncoder(TypeEncoder): + @property + def python_type(self): + return Decimal + + def transform_python(self, value): + return Decimal128(value) + + +class DecimalDecoder(TypeDecoder): + @property + def bson_type(self): + return Decimal128 + + def transform_bson(self, value): + return value.to_decimal() + + +class DecimalCodec(DecimalDecoder, DecimalEncoder): + pass + + +DECIMAL_CODECOPTS = CodecOptions(type_registry=TypeRegistry([DecimalCodec()])) class UndecipherableInt64Type: diff --git a/test/test_encryption.py b/test/test_encryption.py index 46d8c785c4..5c8813203d 100644 --- a/test/test_encryption.py +++ b/test/test_encryption.py @@ -54,14 +54,11 @@ from test import ( unittest, ) -from test.helpers_shared import ( - ALL_KMS_PROVIDERS, +from test.helpers import ( AWS_CREDS, - AWS_TEMP_CREDS, AZURE_CREDS, CA_PEM, CLIENT_PEM, - DEFAULT_KMS_TLS, GCP_CREDS, KMIP_CREDS, LOCAL_MASTER_KEY, @@ -207,7 +204,7 @@ def test_init_kms_tls_options(self): opts = AutoEncryptionOpts( {}, "k.d", - kms_tls_options=DEFAULT_KMS_TLS, + kms_tls_options={"kmip": {"tlsCAFile": CA_PEM, "tlsCertificateKeyFile": CLIENT_PEM}}, ) _kms_ssl_contexts = _parse_kms_tls_options(opts._kms_tls_options, _IS_SYNC) ctx = _kms_ssl_contexts["kmip"] @@ -617,10 +614,17 @@ def test_with_statement(self): # Spec tests +AWS_TEMP_CREDS = { + "accessKeyId": os.environ.get("CSFLE_AWS_TEMP_ACCESS_KEY_ID", ""), + "secretAccessKey": os.environ.get("CSFLE_AWS_TEMP_SECRET_ACCESS_KEY", ""), + "sessionToken": os.environ.get("CSFLE_AWS_TEMP_SESSION_TOKEN", ""), +} + AWS_TEMP_NO_SESSION_CREDS = { "accessKeyId": os.environ.get("CSFLE_AWS_TEMP_ACCESS_KEY_ID", ""), "secretAccessKey": os.environ.get("CSFLE_AWS_TEMP_SECRET_ACCESS_KEY", ""), } +KMS_TLS_OPTS = {"kmip": {"tlsCAFile": CA_PEM, "tlsCertificateKeyFile": CLIENT_PEM}} class TestSpec(SpecRunner): @@ -657,7 +661,7 @@ def parse_auto_encrypt_opts(self, opts): self.skipTest("GCP environment credentials are not set") if "kmip" in kms_providers: kms_providers["kmip"] = KMIP_CREDS - opts["kms_tls_options"] = DEFAULT_KMS_TLS + opts["kms_tls_options"] = KMS_TLS_OPTS if "key_vault_namespace" not in opts: opts["key_vault_namespace"] = "keyvault.datakeys" if "extra_options" in opts: @@ -751,6 +755,14 @@ def run_scenario(self): ) # Prose Tests +ALL_KMS_PROVIDERS = { + "aws": AWS_CREDS, + "azure": AZURE_CREDS, + "gcp": GCP_CREDS, + "kmip": KMIP_CREDS, + "local": {"key": LOCAL_MASTER_KEY}, +} + LOCAL_KEY_ID = Binary(base64.b64decode(b"LOCALAAAAAAAAAAAAAAAAA=="), UUID_SUBTYPE) AWS_KEY_ID = Binary(base64.b64decode(b"AWSAAAAAAAAAAAAAAAAAAA=="), UUID_SUBTYPE) AZURE_KEY_ID = Binary(base64.b64decode(b"AZUREAAAAAAAAAAAAAAAAA=="), UUID_SUBTYPE) @@ -837,17 +849,13 @@ def setUp(self): self.KMS_PROVIDERS, "keyvault.datakeys", schema_map=schemas, - kms_tls_options=DEFAULT_KMS_TLS, + kms_tls_options=KMS_TLS_OPTS, ) self.client_encrypted = self.rs_or_single_client( auto_encryption_opts=opts, uuidRepresentation="standard" ) self.client_encryption = self.create_client_encryption( - self.KMS_PROVIDERS, - "keyvault.datakeys", - self.client, - OPTS, - kms_tls_options=DEFAULT_KMS_TLS, + self.KMS_PROVIDERS, "keyvault.datakeys", self.client, OPTS, kms_tls_options=KMS_TLS_OPTS ) self.listener.reset() @@ -1054,7 +1062,7 @@ def _test_corpus(self, opts): "keyvault.datakeys", client_context.client, OPTS, - kms_tls_options=DEFAULT_KMS_TLS, + kms_tls_options=KMS_TLS_OPTS, ) corpus = self.fix_up_curpus(json_data("corpus", "corpus.json")) @@ -1146,7 +1154,7 @@ def _test_corpus(self, opts): def test_corpus(self): opts = AutoEncryptionOpts( - self.kms_providers(), "keyvault.datakeys", kms_tls_options=DEFAULT_KMS_TLS + self.kms_providers(), "keyvault.datakeys", kms_tls_options=KMS_TLS_OPTS ) self._test_corpus(opts) @@ -1157,7 +1165,7 @@ def test_corpus_local_schema(self): self.kms_providers(), "keyvault.datakeys", schema_map=schemas, - kms_tls_options=DEFAULT_KMS_TLS, + kms_tls_options=KMS_TLS_OPTS, ) self._test_corpus(opts) @@ -1288,7 +1296,7 @@ def setUp(self): key_vault_namespace="keyvault.datakeys", key_vault_client=client_context.client, codec_options=OPTS, - kms_tls_options=DEFAULT_KMS_TLS, + kms_tls_options=KMS_TLS_OPTS, ) kms_providers_invalid = copy.deepcopy(kms_providers) @@ -1300,7 +1308,7 @@ def setUp(self): key_vault_namespace="keyvault.datakeys", key_vault_client=client_context.client, codec_options=OPTS, - kms_tls_options=DEFAULT_KMS_TLS, + kms_tls_options=KMS_TLS_OPTS, ) self._kmip_host_error = None self._invalid_host_error = None @@ -2728,7 +2736,7 @@ def run_test(self, src_provider, dst_provider): key_vault_client=self.client, key_vault_namespace="keyvault.datakeys", kms_providers=ALL_KMS_PROVIDERS, - kms_tls_options=DEFAULT_KMS_TLS, + kms_tls_options=KMS_TLS_OPTS, codec_options=OPTS, ) @@ -2748,7 +2756,7 @@ def run_test(self, src_provider, dst_provider): key_vault_client=client2, key_vault_namespace="keyvault.datakeys", kms_providers=ALL_KMS_PROVIDERS, - kms_tls_options=DEFAULT_KMS_TLS, + kms_tls_options=KMS_TLS_OPTS, codec_options=OPTS, ) diff --git a/test/test_session.py b/test/test_session.py index 40d0a53afb..16a219ae52 100644 --- a/test/test_session.py +++ b/test/test_session.py @@ -378,9 +378,9 @@ def test_cursor_clone(self): with self.client.start_session() as s: cursor = coll.find(session=s) - self.assertIs(cursor.session, s) + self.assertTrue(cursor.session is s) clone = cursor.clone() - self.assertIs(clone.session, s) + self.assertTrue(clone.session is s) # No explicit session. cursor = coll.find(batch_size=2) @@ -392,7 +392,7 @@ def test_cursor_clone(self): next(clone) self.assertIsNone(clone.session) self.assertIsNotNone(clone._session) - self.assertIsNot(cursor._session, clone._session) + self.assertFalse(cursor._session is clone._session) cursor.close() clone.close() diff --git a/test/test_uri_spec.py b/test/test_uri_spec.py index 3d8f7b2b75..8f673cff4c 100644 --- a/test/test_uri_spec.py +++ b/test/test_uri_spec.py @@ -25,7 +25,7 @@ sys.path[0:0] = [""] from test import unittest -from test.helpers_shared import clear_warning_registry +from test.helpers import clear_warning_registry from pymongo.common import INTERNAL_URI_OPTION_NAME_MAP, _CaseInsensitiveDictionary, validate from pymongo.compression_support import _have_snappy diff --git a/test/unified-test-format/valid-pass/poc-queryable-encryption.json b/test/unified-test-format/valid-pass/poc-queryable-encryption.json deleted file mode 100644 index 309d1d3b4b..0000000000 --- a/test/unified-test-format/valid-pass/poc-queryable-encryption.json +++ /dev/null @@ -1,193 +0,0 @@ -{ - "description": "poc-queryable-encryption", - "schemaVersion": "1.23", - "runOnRequirements": [ - { - "minServerVersion": "7.0", - "csfle": true, - "topologies": [ - "replicaset", - "load-balanced", - "sharded" - ] - } - ], - "createEntities": [ - { - "client": { - "id": "client0", - "autoEncryptOpts": { - "keyVaultNamespace": "keyvault.datakeys", - "kmsProviders": { - "local": { - "key": "Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk" - } - } - } - } - }, - { - "database": { - "id": "encryptedDB", - "client": "client0", - "databaseName": "poc-queryable-encryption" - } - }, - { - "collection": { - "id": "encryptedColl", - "database": "encryptedDB", - "collectionName": "encrypted" - } - }, - { - "client": { - "id": "client1" - } - }, - { - "database": { - "id": "unencryptedDB", - "client": "client1", - "databaseName": "poc-queryable-encryption" - } - }, - { - "collection": { - "id": "unencryptedColl", - "database": "unencryptedDB", - "collectionName": "encrypted" - } - } - ], - "initialData": [ - { - "databaseName": "keyvault", - "collectionName": "datakeys", - "documents": [ - { - "_id": { - "$binary": { - "base64": "EjRWeBI0mHYSNBI0VniQEg==", - "subType": "04" - } - }, - "keyMaterial": { - "$binary": { - "base64": "sHe0kz57YW7v8g9VP9sf/+K1ex4JqKc5rf/URX3n3p8XdZ6+15uXPaSayC6adWbNxkFskuMCOifDoTT+rkqMtFkDclOy884RuGGtUysq3X7zkAWYTKi8QAfKkajvVbZl2y23UqgVasdQu3OVBQCrH/xY00nNAs/52e958nVjBuzQkSb1T8pKJAyjZsHJ60+FtnfafDZSTAIBJYn7UWBCwQ==", - "subType": "00" - } - }, - "creationDate": { - "$date": { - "$numberLong": "1641024000000" - } - }, - "updateDate": { - "$date": { - "$numberLong": "1641024000000" - } - }, - "status": 1, - "masterKey": { - "provider": "local" - } - } - ] - }, - { - "databaseName": "poc-queryable-encryption", - "collectionName": "encrypted", - "documents": [], - "createOptions": { - "encryptedFields": { - "fields": [ - { - "keyId": { - "$binary": { - "base64": "EjRWeBI0mHYSNBI0VniQEg==", - "subType": "04" - } - }, - "path": "encryptedInt", - "bsonType": "int", - "queries": { - "queryType": "equality", - "contention": { - "$numberLong": "0" - } - } - } - ] - } - } - } - ], - "tests": [ - { - "description": "insert, replace, and find with queryable encryption", - "operations": [ - { - "object": "encryptedColl", - "name": "insertOne", - "arguments": { - "document": { - "_id": 1, - "encryptedInt": 11 - } - } - }, - { - "object": "encryptedColl", - "name": "replaceOne", - "arguments": { - "filter": { - "encryptedInt": 11 - }, - "replacement": { - "encryptedInt": 22 - } - } - }, - { - "object": "encryptedColl", - "name": "find", - "arguments": { - "filter": { - "encryptedInt": 22 - } - }, - "expectResult": [ - { - "_id": 1, - "encryptedInt": 22 - } - ] - }, - { - "object": "unencryptedColl", - "name": "find", - "arguments": { - "filter": {} - }, - "expectResult": [ - { - "_id": 1, - "encryptedInt": { - "$$type": "binData" - }, - "__safeContent__": [ - { - "$binary": { - "base64": "rhS16TJojgDDBtbluxBokvcotP1mQTGeYpNt8xd3MJQ=", - "subType": "00" - } - } - ] - } - ] - } - ] - } - ] -} diff --git a/test/unified_format.py b/test/unified_format.py index bc21464ab6..3496b2ad44 100644 --- a/test/unified_format.py +++ b/test/unified_format.py @@ -35,7 +35,6 @@ client_knobs, unittest, ) -from test.helpers_shared import ALL_KMS_PROVIDERS, DEFAULT_KMS_TLS from test.unified_format_shared import ( KMS_TLS_OPTS, PLACEHOLDER_MAP, @@ -61,8 +60,6 @@ from test.version import Version from typing import Any, Dict, List, Mapping, Optional -import pytest - import pymongo from bson import SON, json_util from bson.codec_options import DEFAULT_CODEC_OPTIONS @@ -71,7 +68,7 @@ from gridfs.errors import CorruptGridFile from pymongo import ASCENDING, CursorType, MongoClient, _csot from pymongo.driver_info import DriverInfo -from pymongo.encryption_options import _HAVE_PYMONGOCRYPT, AutoEncryptionOpts +from pymongo.encryption_options import _HAVE_PYMONGOCRYPT from pymongo.errors import ( AutoReconnect, BulkWriteError, @@ -158,14 +155,6 @@ def is_run_on_requirement_satisfied(requirement): if req_csfle is True: min_version_satisfied = Version.from_string("4.2") <= server_version csfle_satisfied = _HAVE_PYMONGOCRYPT and min_version_satisfied - elif isinstance(req_csfle, dict) and "minLibmongocryptVersion" in req_csfle: - csfle_satisfied = False - req_version = req_csfle["minLibmongocryptVersion"] - if _HAVE_PYMONGOCRYPT: - from pymongocrypt import libmongocrypt_version - - if Version.from_string(libmongocrypt_version()) >= Version.from_string(req_version): - csfle_satisfied = True return ( topology_satisfied @@ -269,23 +258,6 @@ def _create_entity(self, entity_spec, uri=None): kwargs: dict = {} observe_events = spec.get("observeEvents", []) - if "autoEncryptOpts" in spec: - auto_encrypt_opts = spec["autoEncryptOpts"].copy() - auto_encrypt_kwargs: dict = dict(kms_tls_options=DEFAULT_KMS_TLS) - kms_providers = ALL_KMS_PROVIDERS.copy() - key_vault_namespace = auto_encrypt_opts.pop("keyVaultNamespace") - for provider_name, provider_value in auto_encrypt_opts.pop("kmsProviders").items(): - kms_providers[provider_name].update(provider_value) - extra_opts = auto_encrypt_opts.pop("extraOptions", {}) - for key, value in extra_opts.items(): - auto_encrypt_kwargs[camel_to_snake(key)] = value - for key, value in auto_encrypt_opts.items(): - auto_encrypt_kwargs[camel_to_snake(key)] = value - auto_encryption_opts = AutoEncryptionOpts( - kms_providers, key_vault_namespace, **auto_encrypt_kwargs - ) - kwargs["auto_encryption_opts"] = auto_encryption_opts - # The unified tests use topologyOpeningEvent, we use topologyOpenedEvent for i in range(len(observe_events)): if "topologyOpeningEvent" == observe_events[i]: @@ -457,7 +429,7 @@ class UnifiedSpecTestMixinV1(IntegrationTest): a class attribute ``TEST_SPEC``. """ - SCHEMA_VERSION = Version.from_string("1.25") + SCHEMA_VERSION = Version.from_string("1.22") RUN_ON_LOAD_BALANCER = True TEST_SPEC: Any TEST_PATH = "" # This gets filled in by generate_test_classes @@ -489,13 +461,6 @@ def insert_initial_data(self, initial_data): wc = WriteConcern(w="majority") else: wc = WriteConcern(w=1) - - # Remove any encryption collections associated with the collection. - collections = db.list_collection_names() - for collection in collections: - if collection in [f"enxcol_.{coll_name}.esc", f"enxcol_.{coll_name}.ecoc"]: - db.drop_collection(collection) - if documents: if opts: db.create_collection(coll_name, **opts) @@ -598,6 +563,8 @@ def maybe_skip_test(self, spec): self.skipTest("CSOT not implemented for watch()") if "cursors" in class_name: self.skipTest("CSOT not implemented for cursors") + if "dropindex on collection" in description: + self.skipTest("PYTHON-5491") if ( "tailable" in class_name or "tailable" in description @@ -1534,14 +1501,7 @@ class SpecTestBase(with_metaclass(UnifiedSpecTestMeta)): # type: ignore TEST_SPEC = test_spec EXPECTED_FAILURES = expected_failures - base = SpecTestBase - - # Add "encryption" marker if the "csfle" runOnRequirement is set. - for req in test_spec.get("runOnRequirements", []): - if "csfle" in req: - base = pytest.mark.encryption(base) - - return base + return SpecTestBase for dirpath, _, filenames in os.walk(test_path): dirname = os.path.split(dirpath)[-1] diff --git a/test/unified_format_shared.py b/test/unified_format_shared.py index 96b037976b..17dd73ec8c 100644 --- a/test/unified_format_shared.py +++ b/test/unified_format_shared.py @@ -25,10 +25,9 @@ import time import types from collections import abc -from test.helpers_shared import ( +from test.helpers import ( AWS_CREDS, AWS_CREDS_2, - AWS_TEMP_CREDS, AZURE_CREDS, CA_PEM, CLIENT_PEM, @@ -119,22 +118,10 @@ ("kmip", KMIP_CREDS), ("kmip:name1", KMIP_CREDS), ]: - # Use the temp aws creds for autoEncryptOpts. - if provider_name == "aws": - for key, value in AWS_TEMP_CREDS.items(): - placeholder = f"/autoEncryptOpts/kmsProviders/{provider_name}/{key}" - PLACEHOLDER_MAP[placeholder] = value - for key, value in provider_data.items(): placeholder = f"/clientEncryptionOpts/kmsProviders/{provider_name}/{key}" PLACEHOLDER_MAP[placeholder] = value - if provider_name == "aws": - continue - - placeholder = f"/autoEncryptOpts/kmsProviders/{provider_name}/{key}" - PLACEHOLDER_MAP[placeholder] = value - OIDC_ENV = os.environ.get("OIDC_ENV", "test") if OIDC_ENV == "test": PLACEHOLDER_MAP["/uriOptions/authMechanismProperties"] = {"ENVIRONMENT": "test"} From f1544aa4d52cc9a35b7ba9f9fb5087c0dd097bc1 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 26 Aug 2025 19:47:25 -0500 Subject: [PATCH 18/53] undo topology changes --- pymongo/asynchronous/topology.py | 2 -- pymongo/synchronous/topology.py | 2 -- 2 files changed, 4 deletions(-) diff --git a/pymongo/asynchronous/topology.py b/pymongo/asynchronous/topology.py index fcb9af7ee7..283aabc690 100644 --- a/pymongo/asynchronous/topology.py +++ b/pymongo/asynchronous/topology.py @@ -896,14 +896,12 @@ async def _handle_error(self, address: _Address, err_ctx: _ErrorContext) -> None # ... MUST NOT request an immediate check of the server." if not self._settings.load_balanced: await self._process_change(ServerDescription(address, error=error)) - # Clear the pool. await server.reset(service_id) # "When a client marks a server Unknown from `Network error when # reading or writing`_, clients MUST cancel the hello check on # that server and close the current monitoring connection." server._monitor.cancel_check() - return async def handle_error(self, address: _Address, err_ctx: _ErrorContext) -> None: """Handle an application error. diff --git a/pymongo/synchronous/topology.py b/pymongo/synchronous/topology.py index 57108ad832..a4ca0e6e0f 100644 --- a/pymongo/synchronous/topology.py +++ b/pymongo/synchronous/topology.py @@ -894,14 +894,12 @@ def _handle_error(self, address: _Address, err_ctx: _ErrorContext) -> None: # ... MUST NOT request an immediate check of the server." if not self._settings.load_balanced: self._process_change(ServerDescription(address, error=error)) - # Clear the pool. server.reset(service_id) # "When a client marks a server Unknown from `Network error when # reading or writing`_, clients MUST cancel the hello check on # that server and close the current monitoring connection." server._monitor.cancel_check() - return def handle_error(self, address: _Address, err_ctx: _ErrorContext) -> None: """Handle an application error. From 9d34e527e1fad650f1cbf8652495d5d230647dbe Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 26 Aug 2025 20:28:38 -0500 Subject: [PATCH 19/53] improve sleep translation --- pymongo/synchronous/pool.py | 2 +- tools/synchro.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pymongo/synchronous/pool.py b/pymongo/synchronous/pool.py index 7a529919d2..6411f48e29 100644 --- a/pymongo/synchronous/pool.py +++ b/pymongo/synchronous/pool.py @@ -1054,7 +1054,7 @@ def connect(self, handler: Optional[_MongoClientErrorHandler] = None) -> Connect # Apply backoff if applicable. if self._backoff: - asyncio.sleep(_backoff(self._backoff)) + time.sleep(_backoff(self._backoff)) try: networking_interface = _configured_socket_interface(self.address, self.opts) diff --git a/tools/synchro.py b/tools/synchro.py index 44698134cd..2180260311 100644 --- a/tools/synchro.py +++ b/tools/synchro.py @@ -341,7 +341,7 @@ def translate_async_sleeps(lines: list[str]) -> list[str]: sleeps = [line for line in lines if "asyncio.sleep" in line] for line in sleeps: - res = re.search(r"asyncio.sleep\(([^()]*)\)", line) + res = re.search(r"asyncio\.sleep\s*\(\s*(.*?)\s*\)", line) if res: old = res[0] index = lines.index(line) From bb5ac35e9ed155fd9644e15620ad03c59841a254 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 26 Aug 2025 20:30:05 -0500 Subject: [PATCH 20/53] improve sleep translation --- tools/synchro.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/synchro.py b/tools/synchro.py index 2180260311..968d0e362f 100644 --- a/tools/synchro.py +++ b/tools/synchro.py @@ -341,7 +341,7 @@ def translate_async_sleeps(lines: list[str]) -> list[str]: sleeps = [line for line in lines if "asyncio.sleep" in line] for line in sleeps: - res = re.search(r"asyncio\.sleep\s*\(\s*(.*?)\s*\)", line) + res = re.search(r"asyncio\.sleep\(\s*(.*?)\)", line) if res: old = res[0] index = lines.index(line) From 957a87d35f9a5c154e538fd6a0d3b977a963960c Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 26 Aug 2025 09:52:09 -0500 Subject: [PATCH 21/53] PYTHON-5519 Clean up uv handling (#2510) (cherry picked from commit 0d4c84e86ff8ba7c1ee0e01a1bc2690de502c7e3) --- .evergreen/run-tests.sh | 5 +---- justfile | 26 +++++++++++++++----------- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/.evergreen/run-tests.sh b/.evergreen/run-tests.sh index 2b7d856d41..a9f2ba2b5c 100755 --- a/.evergreen/run-tests.sh +++ b/.evergreen/run-tests.sh @@ -26,12 +26,9 @@ else fi # List the packages. -uv sync ${UV_ARGS} --reinstall +uv sync ${UV_ARGS} --reinstall --quiet uv pip list -# Ensure we go back to base environment after the test. -trap "uv sync" EXIT HUP - # Start the test runner. uv run ${UV_ARGS} .evergreen/scripts/run_tests.py "$@" diff --git a/justfile b/justfile index 74ebb48823..24da94a499 100644 --- a/justfile +++ b/justfile @@ -2,7 +2,7 @@ set shell := ["bash", "-c"] # Commonly used command segments. -uv_run := "uv run --isolated --frozen " +uv_run := "uv run --frozen " typing_run := uv_run + "--group typing --extra aws --extra encryption --extra ocsp --extra snappy --extra test --extra zstd" docs_run := uv_run + "--extra docs" doc_build := "./doc/_build" @@ -13,51 +13,55 @@ mypy_args := "--install-types --non-interactive" default: @just --list +[private] +resync: + @uv sync --quiet --frozen + install: bash .evergreen/scripts/setup-dev-env.sh [group('docs')] -docs: +docs: && resync {{docs_run}} sphinx-build -W -b html doc {{doc_build}}/html [group('docs')] -docs-serve: +docs-serve: && resync {{docs_run}} sphinx-autobuild -W -b html doc --watch ./pymongo --watch ./bson --watch ./gridfs {{doc_build}}/serve [group('docs')] -docs-linkcheck: +docs-linkcheck: && resync {{docs_run}} sphinx-build -E -b linkcheck doc {{doc_build}}/linkcheck [group('typing')] -typing: +typing: && resync just typing-mypy just typing-pyright [group('typing')] -typing-mypy: +typing-mypy: && resync {{typing_run}} mypy {{mypy_args}} bson gridfs tools pymongo {{typing_run}} mypy {{mypy_args}} --config-file mypy_test.ini test {{typing_run}} mypy {{mypy_args}} test/test_typing.py test/test_typing_strict.py [group('typing')] -typing-pyright: +typing-pyright: && resync {{typing_run}} pyright test/test_typing.py test/test_typing_strict.py {{typing_run}} pyright -p strict_pyrightconfig.json test/test_typing_strict.py [group('lint')] -lint: +lint: && resync {{uv_run}} pre-commit run --all-files [group('lint')] -lint-manual: +lint-manual: && resync {{uv_run}} pre-commit run --all-files --hook-stage manual [group('test')] -test *args="-v --durations=5 --maxfail=10": +test *args="-v --durations=5 --maxfail=10": && resync {{uv_run}} --extra test pytest {{args}} [group('test')] -run-tests *args: +run-tests *args: && resync bash ./.evergreen/run-tests.sh {{args}} [group('test')] From 9d0af17b894c712eac3fb9a0e90e6f4b5ba9e40c Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 27 Aug 2025 11:23:23 -0500 Subject: [PATCH 22/53] add prose tests --- test/asynchronous/test_pooling.py | 61 +++++++++++++++++++++++++++++++ test/test_pooling.py | 61 +++++++++++++++++++++++++++++++ 2 files changed, 122 insertions(+) diff --git a/test/asynchronous/test_pooling.py b/test/asynchronous/test_pooling.py index 3193d9e3d5..a26a93a29e 100644 --- a/test/asynchronous/test_pooling.py +++ b/test/asynchronous/test_pooling.py @@ -513,6 +513,67 @@ async def test_connection_timeout_message(self): str(error.exception), ) + @async_client_context.require_failCommand_appName + async def test_pool_backoff_preserves_existing_collections(self): + client = await self.async_rs_or_single_client() + coll = self.db.t + pool = await async_get_pool(client) + await coll.insert_many([{"x": 1} for _ in range(10)]) + t = SocketGetter(self.c, pool) + await t.start() + while t.state != "connection": + await asyncio.sleep(0.1) + + assert not t.sock.conn_closed() + + # Mock a session establishment overload. + mock_connection_fail = { + "configureFailPoint": "failCommand", + "mode": {"times": 1}, + "data": { + "closeConnection": True, + }, + } + + async with self.fail_point(mock_connection_fail): + await coll.find_one({}) + + # Make sure the pool is out of backoff state. + assert pool._backoff == 0 + + # Make sure the existing socket was not affected. + assert not t.sock.conn_closed() + + # Cleanup + await t.release_conn() + await t.join() + await pool.close() + + async def test_pool_check_backoff(self): + # Test that Pool recovers from two connection failures in a row. + # This exercises code at the end of Pool._check(). + cx_pool = await self.create_pool(max_pool_size=1, connect_timeout=1, wait_queue_timeout=1) + self.addAsyncCleanup(cx_pool.close) + + async with cx_pool.checkout() as conn: + # Simulate a closed socket without telling the Connection it's + # closed. + await conn.conn.close() + + # Enable backoff. + cx_pool._backoff = 1 + + # Swap pool's address with a bad one. + address, cx_pool.address = cx_pool.address, ("foo.com", 1234) + with self.assertRaises(AutoReconnect): + async with cx_pool.checkout(): + pass + + # Back to normal, semaphore was correctly released. + cx_pool.address = address + async with cx_pool.checkout(): + pass + class TestPoolMaxSize(_TestPoolingBase): async def test_max_pool_size(self): diff --git a/test/test_pooling.py b/test/test_pooling.py index cb5b206996..3f76814143 100644 --- a/test/test_pooling.py +++ b/test/test_pooling.py @@ -511,6 +511,67 @@ def test_connection_timeout_message(self): str(error.exception), ) + @client_context.require_failCommand_appName + def test_pool_backoff_preserves_existing_collections(self): + client = self.rs_or_single_client() + coll = self.db.t + pool = get_pool(client) + coll.insert_many([{"x": 1} for _ in range(10)]) + t = SocketGetter(self.c, pool) + t.start() + while t.state != "connection": + time.sleep(0.1) + + assert not t.sock.conn_closed() + + # Mock a session establishment overload. + mock_connection_fail = { + "configureFailPoint": "failCommand", + "mode": {"times": 1}, + "data": { + "closeConnection": True, + }, + } + + with self.fail_point(mock_connection_fail): + coll.find_one({}) + + # Make sure the pool is out of backoff state. + assert pool._backoff == 0 + + # Make sure the existing socket was not affected. + assert not t.sock.conn_closed() + + # Cleanup + t.release_conn() + t.join() + pool.close() + + def test_pool_check_backoff(self): + # Test that Pool recovers from two connection failures in a row. + # This exercises code at the end of Pool._check(). + cx_pool = self.create_pool(max_pool_size=1, connect_timeout=1, wait_queue_timeout=1) + self.addCleanup(cx_pool.close) + + with cx_pool.checkout() as conn: + # Simulate a closed socket without telling the Connection it's + # closed. + conn.conn.close() + + # Enable backoff. + cx_pool._backoff = 1 + + # Swap pool's address with a bad one. + address, cx_pool.address = cx_pool.address, ("foo.com", 1234) + with self.assertRaises(AutoReconnect): + with cx_pool.checkout(): + pass + + # Back to normal, semaphore was correctly released. + cx_pool.address = address + with cx_pool.checkout(): + pass + class TestPoolMaxSize(_TestPoolingBase): def test_max_pool_size(self): From da0c0e5cee1843ec8a8b8dd55155ced0728a22af Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 27 Aug 2025 14:16:57 -0500 Subject: [PATCH 23/53] debug --- .evergreen/run-tests.sh | 2 +- test/asynchronous/test_pooling.py | 8 ++++---- test/test_pooling.py | 8 ++++---- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.evergreen/run-tests.sh b/.evergreen/run-tests.sh index a9f2ba2b5c..0fa23d4ef7 100755 --- a/.evergreen/run-tests.sh +++ b/.evergreen/run-tests.sh @@ -30,6 +30,6 @@ uv sync ${UV_ARGS} --reinstall --quiet uv pip list # Start the test runner. -uv run ${UV_ARGS} .evergreen/scripts/run_tests.py "$@" +uv run ${UV_ARGS} .evergreen/scripts/run_tests.py -vv test/asynchronous/test_pooling.py popd diff --git a/test/asynchronous/test_pooling.py b/test/asynchronous/test_pooling.py index a26a93a29e..b89daf23c4 100644 --- a/test/asynchronous/test_pooling.py +++ b/test/asynchronous/test_pooling.py @@ -508,10 +508,10 @@ async def test_connection_timeout_message(self): with self.assertRaises(Exception) as error: await client.admin.command("ping") - self.assertIn( - "(configured timeouts: socketTimeoutMS: 500.0ms, connectTimeoutMS: 500.0ms)", - str(error.exception), - ) + if "(configured timeouts: socketTimeoutMS: 500.0ms, connectTimeoutMS: 500.0ms)" not in str( + error.exception + ): + raise error.exception @async_client_context.require_failCommand_appName async def test_pool_backoff_preserves_existing_collections(self): diff --git a/test/test_pooling.py b/test/test_pooling.py index 3f76814143..22c542ab85 100644 --- a/test/test_pooling.py +++ b/test/test_pooling.py @@ -506,10 +506,10 @@ def test_connection_timeout_message(self): with self.assertRaises(Exception) as error: client.admin.command("ping") - self.assertIn( - "(configured timeouts: socketTimeoutMS: 500.0ms, connectTimeoutMS: 500.0ms)", - str(error.exception), - ) + if "(configured timeouts: socketTimeoutMS: 500.0ms, connectTimeoutMS: 500.0ms)" not in str( + error.exception + ): + raise error.exception @client_context.require_failCommand_appName def test_pool_backoff_preserves_existing_collections(self): From 70b4113e2dade06eab04850b64ba588df4a106b5 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 27 Aug 2025 16:15:10 -0500 Subject: [PATCH 24/53] fix and update tests --- .evergreen/run-tests.sh | 2 +- pymongo/asynchronous/pool.py | 6 +++++- pymongo/synchronous/pool.py | 6 +++++- test/asynchronous/test_pooling.py | 8 ++++---- .../pool-create-min-size-error.json | 10 ++++++---- test/test_pooling.py | 8 ++++---- 6 files changed, 25 insertions(+), 15 deletions(-) diff --git a/.evergreen/run-tests.sh b/.evergreen/run-tests.sh index 0fa23d4ef7..a9f2ba2b5c 100755 --- a/.evergreen/run-tests.sh +++ b/.evergreen/run-tests.sh @@ -30,6 +30,6 @@ uv sync ${UV_ARGS} --reinstall --quiet uv pip list # Start the test runner. -uv run ${UV_ARGS} .evergreen/scripts/run_tests.py -vv test/asynchronous/test_pooling.py +uv run ${UV_ARGS} .evergreen/scripts/run_tests.py "$@" popd diff --git a/pymongo/asynchronous/pool.py b/pymongo/asynchronous/pool.py index bec5e1b201..f644c9ac42 100644 --- a/pymongo/asynchronous/pool.py +++ b/pymongo/asynchronous/pool.py @@ -53,10 +53,12 @@ DocumentTooLarge, ExecutionTimeout, InvalidOperation, + NetworkTimeout, NotPrimaryError, OperationFailure, PyMongoError, WaitQueueTimeoutError, + _OperationCancelled, ) from pymongo.hello import Hello, HelloCompat from pymongo.helpers_shared import _get_timeout_details, format_timeout_details @@ -1107,7 +1109,9 @@ async def connect(self, handler: Optional[_MongoClientErrorHandler] = None) -> A async with self.lock: self.active_contexts.discard(conn.cancel_context) # Enter backoff mode and reconnect on establishment failure. - if isinstance(e, ConnectionFailure): + if isinstance(e, ConnectionFailure) and not isinstance( + e, (_OperationCancelled, NetworkTimeout) + ): self._backoff += 1 # TODO: emit a message about backoff. return await self.connect(handler) diff --git a/pymongo/synchronous/pool.py b/pymongo/synchronous/pool.py index 6411f48e29..9055fde3cb 100644 --- a/pymongo/synchronous/pool.py +++ b/pymongo/synchronous/pool.py @@ -50,10 +50,12 @@ DocumentTooLarge, ExecutionTimeout, InvalidOperation, + NetworkTimeout, NotPrimaryError, OperationFailure, PyMongoError, WaitQueueTimeoutError, + _OperationCancelled, ) from pymongo.hello import Hello, HelloCompat from pymongo.helpers_shared import _get_timeout_details, format_timeout_details @@ -1103,7 +1105,9 @@ def connect(self, handler: Optional[_MongoClientErrorHandler] = None) -> Connect with self.lock: self.active_contexts.discard(conn.cancel_context) # Enter backoff mode and reconnect on establishment failure. - if isinstance(e, ConnectionFailure): + if isinstance(e, ConnectionFailure) and not isinstance( + e, (_OperationCancelled, NetworkTimeout) + ): self._backoff += 1 # TODO: emit a message about backoff. return self.connect(handler) diff --git a/test/asynchronous/test_pooling.py b/test/asynchronous/test_pooling.py index b89daf23c4..a26a93a29e 100644 --- a/test/asynchronous/test_pooling.py +++ b/test/asynchronous/test_pooling.py @@ -508,10 +508,10 @@ async def test_connection_timeout_message(self): with self.assertRaises(Exception) as error: await client.admin.command("ping") - if "(configured timeouts: socketTimeoutMS: 500.0ms, connectTimeoutMS: 500.0ms)" not in str( - error.exception - ): - raise error.exception + self.assertIn( + "(configured timeouts: socketTimeoutMS: 500.0ms, connectTimeoutMS: 500.0ms)", + str(error.exception), + ) @async_client_context.require_failCommand_appName async def test_pool_backoff_preserves_existing_collections(self): diff --git a/test/connection_monitoring/pool-create-min-size-error.json b/test/connection_monitoring/pool-create-min-size-error.json index 1c744b850c..8ec958780d 100644 --- a/test/connection_monitoring/pool-create-min-size-error.json +++ b/test/connection_monitoring/pool-create-min-size-error.json @@ -9,21 +9,23 @@ ], "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 50 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "isMaster", "hello" ], - "closeConnection": true, + "closeConnection": false, + "blockConnection": true, + "blockTimeMS": 1000, "appName": "poolCreateMinSizeErrorTest" } }, "poolOptions": { "minPoolSize": 1, "backgroundThreadIntervalMS": 50, + "socketTimeoutMS": 500, + "connectTimeoutMS": 500, "appName": "poolCreateMinSizeErrorTest" }, "operations": [ diff --git a/test/test_pooling.py b/test/test_pooling.py index 22c542ab85..3f76814143 100644 --- a/test/test_pooling.py +++ b/test/test_pooling.py @@ -506,10 +506,10 @@ def test_connection_timeout_message(self): with self.assertRaises(Exception) as error: client.admin.command("ping") - if "(configured timeouts: socketTimeoutMS: 500.0ms, connectTimeoutMS: 500.0ms)" not in str( - error.exception - ): - raise error.exception + self.assertIn( + "(configured timeouts: socketTimeoutMS: 500.0ms, connectTimeoutMS: 500.0ms)", + str(error.exception), + ) @client_context.require_failCommand_appName def test_pool_backoff_preserves_existing_collections(self): From c974d36af4a10f42da3edfb8d8722fbb6252dc7f Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 27 Aug 2025 17:01:18 -0500 Subject: [PATCH 25/53] fix logic --- pymongo/asynchronous/pool.py | 8 ++------ pymongo/synchronous/pool.py | 8 ++------ 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/pymongo/asynchronous/pool.py b/pymongo/asynchronous/pool.py index f644c9ac42..0d8f7c12f6 100644 --- a/pymongo/asynchronous/pool.py +++ b/pymongo/asynchronous/pool.py @@ -49,16 +49,13 @@ from pymongo.errors import ( # type:ignore[attr-defined] AutoReconnect, ConfigurationError, - ConnectionFailure, DocumentTooLarge, ExecutionTimeout, InvalidOperation, - NetworkTimeout, NotPrimaryError, OperationFailure, PyMongoError, WaitQueueTimeoutError, - _OperationCancelled, ) from pymongo.hello import Hello, HelloCompat from pymongo.helpers_shared import _get_timeout_details, format_timeout_details @@ -1109,11 +1106,10 @@ async def connect(self, handler: Optional[_MongoClientErrorHandler] = None) -> A async with self.lock: self.active_contexts.discard(conn.cancel_context) # Enter backoff mode and reconnect on establishment failure. - if isinstance(e, ConnectionFailure) and not isinstance( - e, (_OperationCancelled, NetworkTimeout) - ): + if type(e) == AutoReconnect: self._backoff += 1 # TODO: emit a message about backoff. + print("backing off", self._backoff) # noqa: T201 return await self.connect(handler) await conn.close_conn(ConnectionClosedReason.ERROR) raise diff --git a/pymongo/synchronous/pool.py b/pymongo/synchronous/pool.py index 9055fde3cb..0509b0f71f 100644 --- a/pymongo/synchronous/pool.py +++ b/pymongo/synchronous/pool.py @@ -46,16 +46,13 @@ from pymongo.errors import ( # type:ignore[attr-defined] AutoReconnect, ConfigurationError, - ConnectionFailure, DocumentTooLarge, ExecutionTimeout, InvalidOperation, - NetworkTimeout, NotPrimaryError, OperationFailure, PyMongoError, WaitQueueTimeoutError, - _OperationCancelled, ) from pymongo.hello import Hello, HelloCompat from pymongo.helpers_shared import _get_timeout_details, format_timeout_details @@ -1105,11 +1102,10 @@ def connect(self, handler: Optional[_MongoClientErrorHandler] = None) -> Connect with self.lock: self.active_contexts.discard(conn.cancel_context) # Enter backoff mode and reconnect on establishment failure. - if isinstance(e, ConnectionFailure) and not isinstance( - e, (_OperationCancelled, NetworkTimeout) - ): + if type(e) == AutoReconnect: self._backoff += 1 # TODO: emit a message about backoff. + print("backing off", self._backoff) # noqa: T201 return self.connect(handler) conn.close_conn(ConnectionClosedReason.ERROR) raise From 845f17a744b12f557d612361099a466492a0411c Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 27 Aug 2025 17:10:15 -0500 Subject: [PATCH 26/53] only backoff if conn is closed --- pymongo/asynchronous/pool.py | 4 ++-- pymongo/synchronous/pool.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pymongo/asynchronous/pool.py b/pymongo/asynchronous/pool.py index 0d8f7c12f6..602f33eafb 100644 --- a/pymongo/asynchronous/pool.py +++ b/pymongo/asynchronous/pool.py @@ -1102,11 +1102,11 @@ async def connect(self, handler: Optional[_MongoClientErrorHandler] = None) -> A await conn.authenticate() # Catch KeyboardInterrupt, CancelledError, etc. and cleanup. - except BaseException as e: + except BaseException: async with self.lock: self.active_contexts.discard(conn.cancel_context) # Enter backoff mode and reconnect on establishment failure. - if type(e) == AutoReconnect: + if conn.conn_closed(): self._backoff += 1 # TODO: emit a message about backoff. print("backing off", self._backoff) # noqa: T201 diff --git a/pymongo/synchronous/pool.py b/pymongo/synchronous/pool.py index 0509b0f71f..5c42857037 100644 --- a/pymongo/synchronous/pool.py +++ b/pymongo/synchronous/pool.py @@ -1098,11 +1098,11 @@ def connect(self, handler: Optional[_MongoClientErrorHandler] = None) -> Connect conn.authenticate() # Catch KeyboardInterrupt, CancelledError, etc. and cleanup. - except BaseException as e: + except BaseException: with self.lock: self.active_contexts.discard(conn.cancel_context) # Enter backoff mode and reconnect on establishment failure. - if type(e) == AutoReconnect: + if conn.conn_closed(): self._backoff += 1 # TODO: emit a message about backoff. print("backing off", self._backoff) # noqa: T201 From 09fc66db7a15462d0995b7d9d3b0e36e41845575 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 27 Aug 2025 19:16:12 -0500 Subject: [PATCH 27/53] use AutoReconnect --- pymongo/asynchronous/pool.py | 5 +- pymongo/synchronous/pool.py | 5 +- uv.lock | 98 +++++++++++++++++++++--------------- 3 files changed, 63 insertions(+), 45 deletions(-) diff --git a/pymongo/asynchronous/pool.py b/pymongo/asynchronous/pool.py index 602f33eafb..8075eb466e 100644 --- a/pymongo/asynchronous/pool.py +++ b/pymongo/asynchronous/pool.py @@ -1102,11 +1102,12 @@ async def connect(self, handler: Optional[_MongoClientErrorHandler] = None) -> A await conn.authenticate() # Catch KeyboardInterrupt, CancelledError, etc. and cleanup. - except BaseException: + except BaseException as e: async with self.lock: self.active_contexts.discard(conn.cancel_context) # Enter backoff mode and reconnect on establishment failure. - if conn.conn_closed(): + if type(e) == AutoReconnect: + await conn.close_conn(ConnectionClosedReason.ERROR) self._backoff += 1 # TODO: emit a message about backoff. print("backing off", self._backoff) # noqa: T201 diff --git a/pymongo/synchronous/pool.py b/pymongo/synchronous/pool.py index 5c42857037..78125ef042 100644 --- a/pymongo/synchronous/pool.py +++ b/pymongo/synchronous/pool.py @@ -1098,11 +1098,12 @@ def connect(self, handler: Optional[_MongoClientErrorHandler] = None) -> Connect conn.authenticate() # Catch KeyboardInterrupt, CancelledError, etc. and cleanup. - except BaseException: + except BaseException as e: with self.lock: self.active_contexts.discard(conn.cancel_context) # Enter backoff mode and reconnect on establishment failure. - if conn.conn_closed(): + if type(e) == AutoReconnect: + conn.close_conn(ConnectionClosedReason.ERROR) self._backoff += 1 # TODO: emit a message about backoff. print("backing off", self._backoff) # noqa: T201 diff --git a/uv.lock b/uv.lock index 9c45c4cdb9..9c74220460 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 2 +revision = 3 requires-python = ">=3.9" resolution-markers = [ "python_full_version == '3.14.*'", @@ -1047,46 +1047,53 @@ dependencies = [ [[package]] name = "mypy" -version = "1.14.1" +version = "1.17.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "mypy-extensions" }, + { name = "pathspec" }, { name = "tomli", marker = "python_full_version < '3.11'" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/b9/eb/2c92d8ea1e684440f54fa49ac5d9a5f19967b7b472a281f419e69a8d228e/mypy-1.14.1.tar.gz", hash = "sha256:7ec88144fe9b510e8475ec2f5f251992690fcf89ccb4500b214b4226abcd32d6", size = 3216051, upload-time = "2024-12-30T16:39:07.335Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9b/7a/87ae2adb31d68402da6da1e5f30c07ea6063e9f09b5e7cfc9dfa44075e74/mypy-1.14.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:52686e37cf13d559f668aa398dd7ddf1f92c5d613e4f8cb262be2fb4fedb0fcb", size = 11211002, upload-time = "2024-12-30T16:37:22.435Z" }, - { url = "https://files.pythonhosted.org/packages/e1/23/eada4c38608b444618a132be0d199b280049ded278b24cbb9d3fc59658e4/mypy-1.14.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:1fb545ca340537d4b45d3eecdb3def05e913299ca72c290326be19b3804b39c0", size = 10358400, upload-time = "2024-12-30T16:37:53.526Z" }, - { url = "https://files.pythonhosted.org/packages/43/c9/d6785c6f66241c62fd2992b05057f404237deaad1566545e9f144ced07f5/mypy-1.14.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:90716d8b2d1f4cd503309788e51366f07c56635a3309b0f6a32547eaaa36a64d", size = 12095172, upload-time = "2024-12-30T16:37:50.332Z" }, - { url = "https://files.pythonhosted.org/packages/c3/62/daa7e787770c83c52ce2aaf1a111eae5893de9e004743f51bfcad9e487ec/mypy-1.14.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2ae753f5c9fef278bcf12e1a564351764f2a6da579d4a81347e1d5a15819997b", size = 12828732, upload-time = "2024-12-30T16:37:29.96Z" }, - { url = "https://files.pythonhosted.org/packages/1b/a2/5fb18318a3637f29f16f4e41340b795da14f4751ef4f51c99ff39ab62e52/mypy-1.14.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:e0fe0f5feaafcb04505bcf439e991c6d8f1bf8b15f12b05feeed96e9e7bf1427", size = 13012197, upload-time = "2024-12-30T16:38:05.037Z" }, - { url = "https://files.pythonhosted.org/packages/28/99/e153ce39105d164b5f02c06c35c7ba958aaff50a2babba7d080988b03fe7/mypy-1.14.1-cp310-cp310-win_amd64.whl", hash = "sha256:7d54bd85b925e501c555a3227f3ec0cfc54ee8b6930bd6141ec872d1c572f81f", size = 9780836, upload-time = "2024-12-30T16:37:19.726Z" }, - { url = "https://files.pythonhosted.org/packages/da/11/a9422850fd506edbcdc7f6090682ecceaf1f87b9dd847f9df79942da8506/mypy-1.14.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f995e511de847791c3b11ed90084a7a0aafdc074ab88c5a9711622fe4751138c", size = 11120432, upload-time = "2024-12-30T16:37:11.533Z" }, - { url = "https://files.pythonhosted.org/packages/b6/9e/47e450fd39078d9c02d620545b2cb37993a8a8bdf7db3652ace2f80521ca/mypy-1.14.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d64169ec3b8461311f8ce2fd2eb5d33e2d0f2c7b49116259c51d0d96edee48d1", size = 10279515, upload-time = "2024-12-30T16:37:40.724Z" }, - { url = "https://files.pythonhosted.org/packages/01/b5/6c8d33bd0f851a7692a8bfe4ee75eb82b6983a3cf39e5e32a5d2a723f0c1/mypy-1.14.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ba24549de7b89b6381b91fbc068d798192b1b5201987070319889e93038967a8", size = 12025791, upload-time = "2024-12-30T16:36:58.73Z" }, - { url = "https://files.pythonhosted.org/packages/f0/4c/e10e2c46ea37cab5c471d0ddaaa9a434dc1d28650078ac1b56c2d7b9b2e4/mypy-1.14.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:183cf0a45457d28ff9d758730cd0210419ac27d4d3f285beda038c9083363b1f", size = 12749203, upload-time = "2024-12-30T16:37:03.741Z" }, - { url = "https://files.pythonhosted.org/packages/88/55/beacb0c69beab2153a0f57671ec07861d27d735a0faff135a494cd4f5020/mypy-1.14.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f2a0ecc86378f45347f586e4163d1769dd81c5a223d577fe351f26b179e148b1", size = 12885900, upload-time = "2024-12-30T16:37:57.948Z" }, - { url = "https://files.pythonhosted.org/packages/a2/75/8c93ff7f315c4d086a2dfcde02f713004357d70a163eddb6c56a6a5eff40/mypy-1.14.1-cp311-cp311-win_amd64.whl", hash = "sha256:ad3301ebebec9e8ee7135d8e3109ca76c23752bac1e717bc84cd3836b4bf3eae", size = 9777869, upload-time = "2024-12-30T16:37:33.428Z" }, - { url = "https://files.pythonhosted.org/packages/43/1b/b38c079609bb4627905b74fc6a49849835acf68547ac33d8ceb707de5f52/mypy-1.14.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:30ff5ef8519bbc2e18b3b54521ec319513a26f1bba19a7582e7b1f58a6e69f14", size = 11266668, upload-time = "2024-12-30T16:38:02.211Z" }, - { url = "https://files.pythonhosted.org/packages/6b/75/2ed0d2964c1ffc9971c729f7a544e9cd34b2cdabbe2d11afd148d7838aa2/mypy-1.14.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:cb9f255c18052343c70234907e2e532bc7e55a62565d64536dbc7706a20b78b9", size = 10254060, upload-time = "2024-12-30T16:37:46.131Z" }, - { url = "https://files.pythonhosted.org/packages/a1/5f/7b8051552d4da3c51bbe8fcafffd76a6823779101a2b198d80886cd8f08e/mypy-1.14.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8b4e3413e0bddea671012b063e27591b953d653209e7a4fa5e48759cda77ca11", size = 11933167, upload-time = "2024-12-30T16:37:43.534Z" }, - { url = "https://files.pythonhosted.org/packages/04/90/f53971d3ac39d8b68bbaab9a4c6c58c8caa4d5fd3d587d16f5927eeeabe1/mypy-1.14.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:553c293b1fbdebb6c3c4030589dab9fafb6dfa768995a453d8a5d3b23784af2e", size = 12864341, upload-time = "2024-12-30T16:37:36.249Z" }, - { url = "https://files.pythonhosted.org/packages/03/d2/8bc0aeaaf2e88c977db41583559319f1821c069e943ada2701e86d0430b7/mypy-1.14.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fad79bfe3b65fe6a1efaed97b445c3d37f7be9fdc348bdb2d7cac75579607c89", size = 12972991, upload-time = "2024-12-30T16:37:06.743Z" }, - { url = "https://files.pythonhosted.org/packages/6f/17/07815114b903b49b0f2cf7499f1c130e5aa459411596668267535fe9243c/mypy-1.14.1-cp312-cp312-win_amd64.whl", hash = "sha256:8fa2220e54d2946e94ab6dbb3ba0a992795bd68b16dc852db33028df2b00191b", size = 9879016, upload-time = "2024-12-30T16:37:15.02Z" }, - { url = "https://files.pythonhosted.org/packages/9e/15/bb6a686901f59222275ab228453de741185f9d54fecbaacec041679496c6/mypy-1.14.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:92c3ed5afb06c3a8e188cb5da4984cab9ec9a77ba956ee419c68a388b4595255", size = 11252097, upload-time = "2024-12-30T16:37:25.144Z" }, - { url = "https://files.pythonhosted.org/packages/f8/b3/8b0f74dfd072c802b7fa368829defdf3ee1566ba74c32a2cb2403f68024c/mypy-1.14.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:dbec574648b3e25f43d23577309b16534431db4ddc09fda50841f1e34e64ed34", size = 10239728, upload-time = "2024-12-30T16:38:08.634Z" }, - { url = "https://files.pythonhosted.org/packages/c5/9b/4fd95ab20c52bb5b8c03cc49169be5905d931de17edfe4d9d2986800b52e/mypy-1.14.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8c6d94b16d62eb3e947281aa7347d78236688e21081f11de976376cf010eb31a", size = 11924965, upload-time = "2024-12-30T16:38:12.132Z" }, - { url = "https://files.pythonhosted.org/packages/56/9d/4a236b9c57f5d8f08ed346914b3f091a62dd7e19336b2b2a0d85485f82ff/mypy-1.14.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d4b19b03fdf54f3c5b2fa474c56b4c13c9dbfb9a2db4370ede7ec11a2c5927d9", size = 12867660, upload-time = "2024-12-30T16:38:17.342Z" }, - { url = "https://files.pythonhosted.org/packages/40/88/a61a5497e2f68d9027de2bb139c7bb9abaeb1be1584649fa9d807f80a338/mypy-1.14.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:0c911fde686394753fff899c409fd4e16e9b294c24bfd5e1ea4675deae1ac6fd", size = 12969198, upload-time = "2024-12-30T16:38:32.839Z" }, - { url = "https://files.pythonhosted.org/packages/54/da/3d6fc5d92d324701b0c23fb413c853892bfe0e1dbe06c9138037d459756b/mypy-1.14.1-cp313-cp313-win_amd64.whl", hash = "sha256:8b21525cb51671219f5307be85f7e646a153e5acc656e5cebf64bfa076c50107", size = 9885276, upload-time = "2024-12-30T16:38:20.828Z" }, - { url = "https://files.pythonhosted.org/packages/ca/1f/186d133ae2514633f8558e78cd658070ba686c0e9275c5a5c24a1e1f0d67/mypy-1.14.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:3888a1816d69f7ab92092f785a462944b3ca16d7c470d564165fe703b0970c35", size = 11200493, upload-time = "2024-12-30T16:38:26.935Z" }, - { url = "https://files.pythonhosted.org/packages/af/fc/4842485d034e38a4646cccd1369f6b1ccd7bc86989c52770d75d719a9941/mypy-1.14.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:46c756a444117c43ee984bd055db99e498bc613a70bbbc120272bd13ca579fbc", size = 10357702, upload-time = "2024-12-30T16:38:50.623Z" }, - { url = "https://files.pythonhosted.org/packages/b4/e6/457b83f2d701e23869cfec013a48a12638f75b9d37612a9ddf99072c1051/mypy-1.14.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:27fc248022907e72abfd8e22ab1f10e903915ff69961174784a3900a8cba9ad9", size = 12091104, upload-time = "2024-12-30T16:38:53.735Z" }, - { url = "https://files.pythonhosted.org/packages/f1/bf/76a569158db678fee59f4fd30b8e7a0d75bcbaeef49edd882a0d63af6d66/mypy-1.14.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:499d6a72fb7e5de92218db961f1a66d5f11783f9ae549d214617edab5d4dbdbb", size = 12830167, upload-time = "2024-12-30T16:38:56.437Z" }, - { url = "https://files.pythonhosted.org/packages/43/bc/0bc6b694b3103de9fed61867f1c8bd33336b913d16831431e7cb48ef1c92/mypy-1.14.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:57961db9795eb566dc1d1b4e9139ebc4c6b0cb6e7254ecde69d1552bf7613f60", size = 13013834, upload-time = "2024-12-30T16:38:59.204Z" }, - { url = "https://files.pythonhosted.org/packages/b0/79/5f5ec47849b6df1e6943d5fd8e6632fbfc04b4fd4acfa5a5a9535d11b4e2/mypy-1.14.1-cp39-cp39-win_amd64.whl", hash = "sha256:07ba89fdcc9451f2ebb02853deb6aaaa3d2239a236669a63ab3801bbf923ef5c", size = 9781231, upload-time = "2024-12-30T16:39:05.124Z" }, - { url = "https://files.pythonhosted.org/packages/a0/b5/32dd67b69a16d088e533962e5044e51004176a9952419de0370cdaead0f8/mypy-1.14.1-py3-none-any.whl", hash = "sha256:b66a60cc4073aeb8ae00057f9c1f64d49e90f918fbcef9a977eb121da8b8f1d1", size = 2752905, upload-time = "2024-12-30T16:38:42.021Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/8e/22/ea637422dedf0bf36f3ef238eab4e455e2a0dcc3082b5cc067615347ab8e/mypy-1.17.1.tar.gz", hash = "sha256:25e01ec741ab5bb3eec8ba9cdb0f769230368a22c959c4937360efb89b7e9f01", size = 3352570, upload-time = "2025-07-31T07:54:19.204Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/77/a9/3d7aa83955617cdf02f94e50aab5c830d205cfa4320cf124ff64acce3a8e/mypy-1.17.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3fbe6d5555bf608c47203baa3e72dbc6ec9965b3d7c318aa9a4ca76f465bd972", size = 11003299, upload-time = "2025-07-31T07:54:06.425Z" }, + { url = "https://files.pythonhosted.org/packages/83/e8/72e62ff837dd5caaac2b4a5c07ce769c8e808a00a65e5d8f94ea9c6f20ab/mypy-1.17.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:80ef5c058b7bce08c83cac668158cb7edea692e458d21098c7d3bce35a5d43e7", size = 10125451, upload-time = "2025-07-31T07:53:52.974Z" }, + { url = "https://files.pythonhosted.org/packages/7d/10/f3f3543f6448db11881776f26a0ed079865926b0c841818ee22de2c6bbab/mypy-1.17.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c4a580f8a70c69e4a75587bd925d298434057fe2a428faaf927ffe6e4b9a98df", size = 11916211, upload-time = "2025-07-31T07:53:18.879Z" }, + { url = "https://files.pythonhosted.org/packages/06/bf/63e83ed551282d67bb3f7fea2cd5561b08d2bb6eb287c096539feb5ddbc5/mypy-1.17.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dd86bb649299f09d987a2eebb4d52d10603224500792e1bee18303bbcc1ce390", size = 12652687, upload-time = "2025-07-31T07:53:30.544Z" }, + { url = "https://files.pythonhosted.org/packages/69/66/68f2eeef11facf597143e85b694a161868b3b006a5fbad50e09ea117ef24/mypy-1.17.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:a76906f26bd8d51ea9504966a9c25419f2e668f012e0bdf3da4ea1526c534d94", size = 12896322, upload-time = "2025-07-31T07:53:50.74Z" }, + { url = "https://files.pythonhosted.org/packages/a3/87/8e3e9c2c8bd0d7e071a89c71be28ad088aaecbadf0454f46a540bda7bca6/mypy-1.17.1-cp310-cp310-win_amd64.whl", hash = "sha256:e79311f2d904ccb59787477b7bd5d26f3347789c06fcd7656fa500875290264b", size = 9507962, upload-time = "2025-07-31T07:53:08.431Z" }, + { url = "https://files.pythonhosted.org/packages/46/cf/eadc80c4e0a70db1c08921dcc220357ba8ab2faecb4392e3cebeb10edbfa/mypy-1.17.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ad37544be07c5d7fba814eb370e006df58fed8ad1ef33ed1649cb1889ba6ff58", size = 10921009, upload-time = "2025-07-31T07:53:23.037Z" }, + { url = "https://files.pythonhosted.org/packages/5d/c1/c869d8c067829ad30d9bdae051046561552516cfb3a14f7f0347b7d973ee/mypy-1.17.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:064e2ff508e5464b4bd807a7c1625bc5047c5022b85c70f030680e18f37273a5", size = 10047482, upload-time = "2025-07-31T07:53:26.151Z" }, + { url = "https://files.pythonhosted.org/packages/98/b9/803672bab3fe03cee2e14786ca056efda4bb511ea02dadcedde6176d06d0/mypy-1.17.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:70401bbabd2fa1aa7c43bb358f54037baf0586f41e83b0ae67dd0534fc64edfd", size = 11832883, upload-time = "2025-07-31T07:53:47.948Z" }, + { url = "https://files.pythonhosted.org/packages/88/fb/fcdac695beca66800918c18697b48833a9a6701de288452b6715a98cfee1/mypy-1.17.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e92bdc656b7757c438660f775f872a669b8ff374edc4d18277d86b63edba6b8b", size = 12566215, upload-time = "2025-07-31T07:54:04.031Z" }, + { url = "https://files.pythonhosted.org/packages/7f/37/a932da3d3dace99ee8eb2043b6ab03b6768c36eb29a02f98f46c18c0da0e/mypy-1.17.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c1fdf4abb29ed1cb091cf432979e162c208a5ac676ce35010373ff29247bcad5", size = 12751956, upload-time = "2025-07-31T07:53:36.263Z" }, + { url = "https://files.pythonhosted.org/packages/8c/cf/6438a429e0f2f5cab8bc83e53dbebfa666476f40ee322e13cac5e64b79e7/mypy-1.17.1-cp311-cp311-win_amd64.whl", hash = "sha256:ff2933428516ab63f961644bc49bc4cbe42bbffb2cd3b71cc7277c07d16b1a8b", size = 9507307, upload-time = "2025-07-31T07:53:59.734Z" }, + { url = "https://files.pythonhosted.org/packages/17/a2/7034d0d61af8098ec47902108553122baa0f438df8a713be860f7407c9e6/mypy-1.17.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:69e83ea6553a3ba79c08c6e15dbd9bfa912ec1e493bf75489ef93beb65209aeb", size = 11086295, upload-time = "2025-07-31T07:53:28.124Z" }, + { url = "https://files.pythonhosted.org/packages/14/1f/19e7e44b594d4b12f6ba8064dbe136505cec813549ca3e5191e40b1d3cc2/mypy-1.17.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1b16708a66d38abb1e6b5702f5c2c87e133289da36f6a1d15f6a5221085c6403", size = 10112355, upload-time = "2025-07-31T07:53:21.121Z" }, + { url = "https://files.pythonhosted.org/packages/5b/69/baa33927e29e6b4c55d798a9d44db5d394072eef2bdc18c3e2048c9ed1e9/mypy-1.17.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:89e972c0035e9e05823907ad5398c5a73b9f47a002b22359b177d40bdaee7056", size = 11875285, upload-time = "2025-07-31T07:53:55.293Z" }, + { url = "https://files.pythonhosted.org/packages/90/13/f3a89c76b0a41e19490b01e7069713a30949d9a6c147289ee1521bcea245/mypy-1.17.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:03b6d0ed2b188e35ee6d5c36b5580cffd6da23319991c49ab5556c023ccf1341", size = 12737895, upload-time = "2025-07-31T07:53:43.623Z" }, + { url = "https://files.pythonhosted.org/packages/23/a1/c4ee79ac484241301564072e6476c5a5be2590bc2e7bfd28220033d2ef8f/mypy-1.17.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c837b896b37cd103570d776bda106eabb8737aa6dd4f248451aecf53030cdbeb", size = 12931025, upload-time = "2025-07-31T07:54:17.125Z" }, + { url = "https://files.pythonhosted.org/packages/89/b8/7409477be7919a0608900e6320b155c72caab4fef46427c5cc75f85edadd/mypy-1.17.1-cp312-cp312-win_amd64.whl", hash = "sha256:665afab0963a4b39dff7c1fa563cc8b11ecff7910206db4b2e64dd1ba25aed19", size = 9584664, upload-time = "2025-07-31T07:54:12.842Z" }, + { url = "https://files.pythonhosted.org/packages/5b/82/aec2fc9b9b149f372850291827537a508d6c4d3664b1750a324b91f71355/mypy-1.17.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:93378d3203a5c0800c6b6d850ad2f19f7a3cdf1a3701d3416dbf128805c6a6a7", size = 11075338, upload-time = "2025-07-31T07:53:38.873Z" }, + { url = "https://files.pythonhosted.org/packages/07/ac/ee93fbde9d2242657128af8c86f5d917cd2887584cf948a8e3663d0cd737/mypy-1.17.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:15d54056f7fe7a826d897789f53dd6377ec2ea8ba6f776dc83c2902b899fee81", size = 10113066, upload-time = "2025-07-31T07:54:14.707Z" }, + { url = "https://files.pythonhosted.org/packages/5a/68/946a1e0be93f17f7caa56c45844ec691ca153ee8b62f21eddda336a2d203/mypy-1.17.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:209a58fed9987eccc20f2ca94afe7257a8f46eb5df1fb69958650973230f91e6", size = 11875473, upload-time = "2025-07-31T07:53:14.504Z" }, + { url = "https://files.pythonhosted.org/packages/9f/0f/478b4dce1cb4f43cf0f0d00fba3030b21ca04a01b74d1cd272a528cf446f/mypy-1.17.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:099b9a5da47de9e2cb5165e581f158e854d9e19d2e96b6698c0d64de911dd849", size = 12744296, upload-time = "2025-07-31T07:53:03.896Z" }, + { url = "https://files.pythonhosted.org/packages/ca/70/afa5850176379d1b303f992a828de95fc14487429a7139a4e0bdd17a8279/mypy-1.17.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:fa6ffadfbe6994d724c5a1bb6123a7d27dd68fc9c059561cd33b664a79578e14", size = 12914657, upload-time = "2025-07-31T07:54:08.576Z" }, + { url = "https://files.pythonhosted.org/packages/53/f9/4a83e1c856a3d9c8f6edaa4749a4864ee98486e9b9dbfbc93842891029c2/mypy-1.17.1-cp313-cp313-win_amd64.whl", hash = "sha256:9a2b7d9180aed171f033c9f2fc6c204c1245cf60b0cb61cf2e7acc24eea78e0a", size = 9593320, upload-time = "2025-07-31T07:53:01.341Z" }, + { url = "https://files.pythonhosted.org/packages/38/56/79c2fac86da57c7d8c48622a05873eaab40b905096c33597462713f5af90/mypy-1.17.1-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:15a83369400454c41ed3a118e0cc58bd8123921a602f385cb6d6ea5df050c733", size = 11040037, upload-time = "2025-07-31T07:54:10.942Z" }, + { url = "https://files.pythonhosted.org/packages/4d/c3/adabe6ff53638e3cad19e3547268482408323b1e68bf082c9119000cd049/mypy-1.17.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:55b918670f692fc9fba55c3298d8a3beae295c5cded0a55dccdc5bbead814acd", size = 10131550, upload-time = "2025-07-31T07:53:41.307Z" }, + { url = "https://files.pythonhosted.org/packages/b8/c5/2e234c22c3bdeb23a7817af57a58865a39753bde52c74e2c661ee0cfc640/mypy-1.17.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:62761474061feef6f720149d7ba876122007ddc64adff5ba6f374fda35a018a0", size = 11872963, upload-time = "2025-07-31T07:53:16.878Z" }, + { url = "https://files.pythonhosted.org/packages/ab/26/c13c130f35ca8caa5f2ceab68a247775648fdcd6c9a18f158825f2bc2410/mypy-1.17.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c49562d3d908fd49ed0938e5423daed8d407774a479b595b143a3d7f87cdae6a", size = 12710189, upload-time = "2025-07-31T07:54:01.962Z" }, + { url = "https://files.pythonhosted.org/packages/82/df/c7d79d09f6de8383fe800521d066d877e54d30b4fb94281c262be2df84ef/mypy-1.17.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:397fba5d7616a5bc60b45c7ed204717eaddc38f826e3645402c426057ead9a91", size = 12900322, upload-time = "2025-07-31T07:53:10.551Z" }, + { url = "https://files.pythonhosted.org/packages/b8/98/3d5a48978b4f708c55ae832619addc66d677f6dc59f3ebad71bae8285ca6/mypy-1.17.1-cp314-cp314-win_amd64.whl", hash = "sha256:9d6b20b97d373f41617bd0708fd46aa656059af57f2ef72aa8c7d6a2b73b74ed", size = 9751879, upload-time = "2025-07-31T07:52:56.683Z" }, + { url = "https://files.pythonhosted.org/packages/29/cb/673e3d34e5d8de60b3a61f44f80150a738bff568cd6b7efb55742a605e98/mypy-1.17.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5d1092694f166a7e56c805caaf794e0585cabdbf1df36911c414e4e9abb62ae9", size = 10992466, upload-time = "2025-07-31T07:53:57.574Z" }, + { url = "https://files.pythonhosted.org/packages/0c/d0/fe1895836eea3a33ab801561987a10569df92f2d3d4715abf2cfeaa29cb2/mypy-1.17.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:79d44f9bfb004941ebb0abe8eff6504223a9c1ac51ef967d1263c6572bbebc99", size = 10117638, upload-time = "2025-07-31T07:53:34.256Z" }, + { url = "https://files.pythonhosted.org/packages/97/f3/514aa5532303aafb95b9ca400a31054a2bd9489de166558c2baaeea9c522/mypy-1.17.1-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b01586eed696ec905e61bd2568f48740f7ac4a45b3a468e6423a03d3788a51a8", size = 11915673, upload-time = "2025-07-31T07:52:59.361Z" }, + { url = "https://files.pythonhosted.org/packages/ab/c3/c0805f0edec96fe8e2c048b03769a6291523d509be8ee7f56ae922fa3882/mypy-1.17.1-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:43808d9476c36b927fbcd0b0255ce75efe1b68a080154a38ae68a7e62de8f0f8", size = 12649022, upload-time = "2025-07-31T07:53:45.92Z" }, + { url = "https://files.pythonhosted.org/packages/45/3e/d646b5a298ada21a8512fa7e5531f664535a495efa672601702398cea2b4/mypy-1.17.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:feb8cc32d319edd5859da2cc084493b3e2ce5e49a946377663cc90f6c15fb259", size = 12895536, upload-time = "2025-07-31T07:53:06.17Z" }, + { url = "https://files.pythonhosted.org/packages/14/55/e13d0dcd276975927d1f4e9e2ec4fd409e199f01bdc671717e673cc63a22/mypy-1.17.1-cp39-cp39-win_amd64.whl", hash = "sha256:d7598cf74c3e16539d4e2f0b8d8c318e00041553d83d4861f87c7a72e95ac24d", size = 9512564, upload-time = "2025-07-31T07:53:12.346Z" }, + { url = "https://files.pythonhosted.org/packages/1d/f3/8fcd2af0f5b806f6cf463efaffd3c9548a28f84220493ecd38d127b6b66d/mypy-1.17.1-py3-none-any.whl", hash = "sha256:a9f52c0351c21fe24c21d8c0eb1f62967b262d6729393397b6f443c3b773c3b9", size = 2283411, upload-time = "2025-07-31T07:53:24.664Z" }, ] [[package]] @@ -1116,6 +1123,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484", size = 66469, upload-time = "2025-04-19T11:48:57.875Z" }, ] +[[package]] +name = "pathspec" +version = "0.12.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ca/bc/f35b8446f4531a7cb215605d100cd88b7ac6f44ab3fc94870c120ab3adbf/pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712", size = 51043, upload-time = "2023-12-10T22:30:45Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cc/20/ff623b09d963f88bfde16306a54e12ee5ea43e9b597108672ff3a408aad6/pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08", size = 31191, upload-time = "2023-12-10T22:30:43.14Z" }, +] + [[package]] name = "pip" version = "25.2" @@ -1315,7 +1331,7 @@ provides-extras = ["aws", "docs", "encryption", "gssapi", "ocsp", "snappy", "tes [package.metadata.requires-dev] coverage = [ - { name = "coverage", specifier = ">=5,<=7.5" }, + { name = "coverage", specifier = ">=5,<=7.10.3" }, { name = "pytest-cov" }, ] dev = [{ name = "pre-commit", specifier = ">=4.0" }] @@ -1329,9 +1345,9 @@ perf = [{ name = "simplejson" }] pip = [{ name = "pip" }] pymongocrypt-source = [{ name = "pymongocrypt", git = "https://github.com/mongodb/libmongocrypt?subdirectory=bindings%2Fpython&rev=master" }] typing = [ - { name = "mypy", specifier = "==1.14.1" }, + { name = "mypy", specifier = "==1.17.1" }, { name = "pip" }, - { name = "pyright", specifier = "==1.1.392.post0" }, + { name = "pyright", specifier = "==1.1.403" }, { name = "typing-extensions" }, ] @@ -1375,15 +1391,15 @@ wheels = [ [[package]] name = "pyright" -version = "1.1.392.post0" +version = "1.1.403" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "nodeenv" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/66/df/3c6f6b08fba7ccf49b114dfc4bb33e25c299883fd763f93fad47ef8bc58d/pyright-1.1.392.post0.tar.gz", hash = "sha256:3b7f88de74a28dcfa90c7d90c782b6569a48c2be5f9d4add38472bdaac247ebd", size = 3789911, upload-time = "2025-01-15T15:01:20.913Z" } +sdist = { url = "https://files.pythonhosted.org/packages/fe/f6/35f885264ff08c960b23d1542038d8da86971c5d8c955cfab195a4f672d7/pyright-1.1.403.tar.gz", hash = "sha256:3ab69b9f41c67fb5bbb4d7a36243256f0d549ed3608678d381d5f51863921104", size = 3913526, upload-time = "2025-07-09T07:15:52.882Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/e7/b1/a18de17f40e4f61ca58856b9ef9b0febf74ff88978c3f7776f910071f567/pyright-1.1.392.post0-py3-none-any.whl", hash = "sha256:252f84458a46fa2f0fd4e2f91fc74f50b9ca52c757062e93f6c250c0d8329eb2", size = 5595487, upload-time = "2025-01-15T15:01:17.775Z" }, + { url = "https://files.pythonhosted.org/packages/49/b6/b04e5c2f41a5ccad74a1a4759da41adb20b4bc9d59a5e08d29ba60084d07/pyright-1.1.403-py3-none-any.whl", hash = "sha256:c0eeca5aa76cbef3fcc271259bbd785753c7ad7bcac99a9162b4c4c7daed23b3", size = 5684504, upload-time = "2025-07-09T07:15:50.958Z" }, ] [[package]] From 84478d0dd21420b968b5e966947fa2b3d70da963 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 27 Aug 2025 19:34:43 -0500 Subject: [PATCH 28/53] update tests --- test/connection_logging/connection-logging.json | 8 ++++++-- .../unified/auth-network-error.json | 6 +++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/test/connection_logging/connection-logging.json b/test/connection_logging/connection-logging.json index 5799e834d7..60190c7dc0 100644 --- a/test/connection_logging/connection-logging.json +++ b/test/connection_logging/connection-logging.json @@ -331,7 +331,9 @@ "uriOptions": { "retryReads": false, "appname": "clientAppName", - "heartbeatFrequencyMS": 10000 + "heartbeatFrequencyMS": 10000, + "socketTimeoutMS": 500, + "connectTimeoutMS": 500 }, "observeLogMessages": { "connection": "debug" @@ -355,7 +357,9 @@ "failCommands": [ "saslContinue" ], - "closeConnection": true, + "closeConnection": false, + "blockConnection": true, + "blockTimeMS": 1000, "appName": "clientAppName" } } diff --git a/test/discovery_and_monitoring/unified/auth-network-error.json b/test/discovery_and_monitoring/unified/auth-network-error.json index 84763af32e..656b291366 100644 --- a/test/discovery_and_monitoring/unified/auth-network-error.json +++ b/test/discovery_and_monitoring/unified/auth-network-error.json @@ -53,7 +53,9 @@ "failCommands": [ "saslContinue" ], - "closeConnection": true, + "closeConnection": false, + "blockConnection": true, + "blockTimeMS": 1000, "appName": "authNetworkErrorTest" } } @@ -75,6 +77,8 @@ ], "uriOptions": { "retryWrites": false, + "socketTimeoutMS": 500, + "connectTimeoutMS": 500, "appname": "authNetworkErrorTest" } } From 532c1b8cba86e037f602b54c762e786ef9786fc4 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 28 Aug 2025 06:17:28 -0500 Subject: [PATCH 29/53] update handshake error tests --- .../unified/handshakeError.json | 96 ------------------- .../unified/handshakeError.json | 60 ------------ 2 files changed, 156 deletions(-) diff --git a/test/retryable_reads/unified/handshakeError.json b/test/retryable_reads/unified/handshakeError.json index 2921d8a954..6a58084f6d 100644 --- a/test/retryable_reads/unified/handshakeError.json +++ b/test/retryable_reads/unified/handshakeError.json @@ -116,9 +116,6 @@ { "connectionCheckOutStartedEvent": {} }, - { - "connectionCheckOutStartedEvent": {} - }, { "connectionCheckOutStartedEvent": {} } @@ -209,9 +206,6 @@ { "connectionCheckOutStartedEvent": {} }, - { - "connectionCheckOutStartedEvent": {} - }, { "connectionCheckOutStartedEvent": {} } @@ -299,9 +293,6 @@ { "connectionCheckOutStartedEvent": {} }, - { - "connectionCheckOutStartedEvent": {} - }, { "connectionCheckOutStartedEvent": {} } @@ -389,9 +380,6 @@ { "connectionCheckOutStartedEvent": {} }, - { - "connectionCheckOutStartedEvent": {} - }, { "connectionCheckOutStartedEvent": {} } @@ -488,9 +476,6 @@ { "connectionCheckOutStartedEvent": {} }, - { - "connectionCheckOutStartedEvent": {} - }, { "connectionCheckOutStartedEvent": {} } @@ -587,9 +572,6 @@ { "connectionCheckOutStartedEvent": {} }, - { - "connectionCheckOutStartedEvent": {} - }, { "connectionCheckOutStartedEvent": {} } @@ -692,9 +674,6 @@ { "connectionCheckOutStartedEvent": {} }, - { - "connectionCheckOutStartedEvent": {} - }, { "connectionCheckOutStartedEvent": {} } @@ -797,9 +776,6 @@ { "connectionCheckOutStartedEvent": {} }, - { - "connectionCheckOutStartedEvent": {} - }, { "connectionCheckOutStartedEvent": {} } @@ -890,9 +866,6 @@ { "connectionCheckOutStartedEvent": {} }, - { - "connectionCheckOutStartedEvent": {} - }, { "connectionCheckOutStartedEvent": {} } @@ -983,9 +956,6 @@ { "connectionCheckOutStartedEvent": {} }, - { - "connectionCheckOutStartedEvent": {} - }, { "connectionCheckOutStartedEvent": {} } @@ -1076,9 +1046,6 @@ { "connectionCheckOutStartedEvent": {} }, - { - "connectionCheckOutStartedEvent": {} - }, { "connectionCheckOutStartedEvent": {} } @@ -1169,9 +1136,6 @@ { "connectionCheckOutStartedEvent": {} }, - { - "connectionCheckOutStartedEvent": {} - }, { "connectionCheckOutStartedEvent": {} } @@ -1268,9 +1232,6 @@ { "connectionCheckOutStartedEvent": {} }, - { - "connectionCheckOutStartedEvent": {} - }, { "connectionCheckOutStartedEvent": {} } @@ -1367,9 +1328,6 @@ { "connectionCheckOutStartedEvent": {} }, - { - "connectionCheckOutStartedEvent": {} - }, { "connectionCheckOutStartedEvent": {} } @@ -1460,9 +1418,6 @@ { "connectionCheckOutStartedEvent": {} }, - { - "connectionCheckOutStartedEvent": {} - }, { "connectionCheckOutStartedEvent": {} } @@ -1553,9 +1508,6 @@ { "connectionCheckOutStartedEvent": {} }, - { - "connectionCheckOutStartedEvent": {} - }, { "connectionCheckOutStartedEvent": {} } @@ -1646,9 +1598,6 @@ { "connectionCheckOutStartedEvent": {} }, - { - "connectionCheckOutStartedEvent": {} - }, { "connectionCheckOutStartedEvent": {} } @@ -1739,9 +1688,6 @@ { "connectionCheckOutStartedEvent": {} }, - { - "connectionCheckOutStartedEvent": {} - }, { "connectionCheckOutStartedEvent": {} } @@ -1829,9 +1775,6 @@ { "connectionCheckOutStartedEvent": {} }, - { - "connectionCheckOutStartedEvent": {} - }, { "connectionCheckOutStartedEvent": {} } @@ -1919,9 +1862,6 @@ { "connectionCheckOutStartedEvent": {} }, - { - "connectionCheckOutStartedEvent": {} - }, { "connectionCheckOutStartedEvent": {} } @@ -2013,9 +1953,6 @@ { "connectionCheckOutStartedEvent": {} }, - { - "connectionCheckOutStartedEvent": {} - }, { "connectionCheckOutStartedEvent": {} } @@ -2107,9 +2044,6 @@ { "connectionCheckOutStartedEvent": {} }, - { - "connectionCheckOutStartedEvent": {} - }, { "connectionCheckOutStartedEvent": {} } @@ -2200,9 +2134,6 @@ { "connectionCheckOutStartedEvent": {} }, - { - "connectionCheckOutStartedEvent": {} - }, { "connectionCheckOutStartedEvent": {} } @@ -2293,9 +2224,6 @@ { "connectionCheckOutStartedEvent": {} }, - { - "connectionCheckOutStartedEvent": {} - }, { "connectionCheckOutStartedEvent": {} } @@ -2386,9 +2314,6 @@ { "connectionCheckOutStartedEvent": {} }, - { - "connectionCheckOutStartedEvent": {} - }, { "connectionCheckOutStartedEvent": {} } @@ -2479,9 +2404,6 @@ { "connectionCheckOutStartedEvent": {} }, - { - "connectionCheckOutStartedEvent": {} - }, { "connectionCheckOutStartedEvent": {} } @@ -2569,9 +2491,6 @@ { "connectionCheckOutStartedEvent": {} }, - { - "connectionCheckOutStartedEvent": {} - }, { "connectionCheckOutStartedEvent": {} } @@ -2659,9 +2578,6 @@ { "connectionCheckOutStartedEvent": {} }, - { - "connectionCheckOutStartedEvent": {} - }, { "connectionCheckOutStartedEvent": {} } @@ -2749,9 +2665,6 @@ { "connectionCheckOutStartedEvent": {} }, - { - "connectionCheckOutStartedEvent": {} - }, { "connectionCheckOutStartedEvent": {} } @@ -2839,9 +2752,6 @@ { "connectionCheckOutStartedEvent": {} }, - { - "connectionCheckOutStartedEvent": {} - }, { "connectionCheckOutStartedEvent": {} } @@ -2938,9 +2848,6 @@ { "connectionCheckOutStartedEvent": {} }, - { - "connectionCheckOutStartedEvent": {} - }, { "connectionCheckOutStartedEvent": {} } @@ -3037,9 +2944,6 @@ { "connectionCheckOutStartedEvent": {} }, - { - "connectionCheckOutStartedEvent": {} - }, { "connectionCheckOutStartedEvent": {} } diff --git a/test/retryable_writes/unified/handshakeError.json b/test/retryable_writes/unified/handshakeError.json index 93cb2e849e..bcda0a0bfc 100644 --- a/test/retryable_writes/unified/handshakeError.json +++ b/test/retryable_writes/unified/handshakeError.json @@ -124,9 +124,6 @@ { "connectionCheckOutStartedEvent": {} }, - { - "connectionCheckOutStartedEvent": {} - }, { "connectionCheckOutStartedEvent": {} } @@ -233,9 +230,6 @@ { "connectionCheckOutStartedEvent": {} }, - { - "connectionCheckOutStartedEvent": {} - }, { "connectionCheckOutStartedEvent": {} } @@ -329,9 +323,6 @@ { "connectionCheckOutStartedEvent": {} }, - { - "connectionCheckOutStartedEvent": {} - }, { "connectionCheckOutStartedEvent": {} } @@ -425,9 +416,6 @@ { "connectionCheckOutStartedEvent": {} }, - { - "connectionCheckOutStartedEvent": {} - }, { "connectionCheckOutStartedEvent": {} } @@ -523,9 +511,6 @@ { "connectionCheckOutStartedEvent": {} }, - { - "connectionCheckOutStartedEvent": {} - }, { "connectionCheckOutStartedEvent": {} } @@ -621,9 +606,6 @@ { "connectionCheckOutStartedEvent": {} }, - { - "connectionCheckOutStartedEvent": {} - }, { "connectionCheckOutStartedEvent": {} } @@ -714,9 +696,6 @@ { "connectionCheckOutStartedEvent": {} }, - { - "connectionCheckOutStartedEvent": {} - }, { "connectionCheckOutStartedEvent": {} } @@ -807,9 +786,6 @@ { "connectionCheckOutStartedEvent": {} }, - { - "connectionCheckOutStartedEvent": {} - }, { "connectionCheckOutStartedEvent": {} } @@ -903,9 +879,6 @@ { "connectionCheckOutStartedEvent": {} }, - { - "connectionCheckOutStartedEvent": {} - }, { "connectionCheckOutStartedEvent": {} } @@ -999,9 +972,6 @@ { "connectionCheckOutStartedEvent": {} }, - { - "connectionCheckOutStartedEvent": {} - }, { "connectionCheckOutStartedEvent": {} } @@ -1097,9 +1067,6 @@ { "connectionCheckOutStartedEvent": {} }, - { - "connectionCheckOutStartedEvent": {} - }, { "connectionCheckOutStartedEvent": {} } @@ -1195,9 +1162,6 @@ { "connectionCheckOutStartedEvent": {} }, - { - "connectionCheckOutStartedEvent": {} - }, { "connectionCheckOutStartedEvent": {} } @@ -1288,9 +1252,6 @@ { "connectionCheckOutStartedEvent": {} }, - { - "connectionCheckOutStartedEvent": {} - }, { "connectionCheckOutStartedEvent": {} } @@ -1381,9 +1342,6 @@ { "connectionCheckOutStartedEvent": {} }, - { - "connectionCheckOutStartedEvent": {} - }, { "connectionCheckOutStartedEvent": {} } @@ -1477,9 +1435,6 @@ { "connectionCheckOutStartedEvent": {} }, - { - "connectionCheckOutStartedEvent": {} - }, { "connectionCheckOutStartedEvent": {} } @@ -1573,9 +1528,6 @@ { "connectionCheckOutStartedEvent": {} }, - { - "connectionCheckOutStartedEvent": {} - }, { "connectionCheckOutStartedEvent": {} } @@ -1671,9 +1623,6 @@ { "connectionCheckOutStartedEvent": {} }, - { - "connectionCheckOutStartedEvent": {} - }, { "connectionCheckOutStartedEvent": {} } @@ -1769,9 +1718,6 @@ { "connectionCheckOutStartedEvent": {} }, - { - "connectionCheckOutStartedEvent": {} - }, { "connectionCheckOutStartedEvent": {} } @@ -1871,9 +1817,6 @@ { "connectionCheckOutStartedEvent": {} }, - { - "connectionCheckOutStartedEvent": {} - }, { "connectionCheckOutStartedEvent": {} } @@ -1973,9 +1916,6 @@ { "connectionCheckOutStartedEvent": {} }, - { - "connectionCheckOutStartedEvent": {} - }, { "connectionCheckOutStartedEvent": {} } From 6890c732edc0a1efae9ab1c6a10d419f765ffa64 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 28 Aug 2025 06:21:27 -0500 Subject: [PATCH 30/53] undo lock file changes --- uv.lock | 98 ++++++++++++++++++++++++--------------------------------- 1 file changed, 41 insertions(+), 57 deletions(-) diff --git a/uv.lock b/uv.lock index 9c74220460..9c45c4cdb9 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 3 +revision = 2 requires-python = ">=3.9" resolution-markers = [ "python_full_version == '3.14.*'", @@ -1047,53 +1047,46 @@ dependencies = [ [[package]] name = "mypy" -version = "1.17.1" +version = "1.14.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "mypy-extensions" }, - { name = "pathspec" }, { name = "tomli", marker = "python_full_version < '3.11'" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/8e/22/ea637422dedf0bf36f3ef238eab4e455e2a0dcc3082b5cc067615347ab8e/mypy-1.17.1.tar.gz", hash = "sha256:25e01ec741ab5bb3eec8ba9cdb0f769230368a22c959c4937360efb89b7e9f01", size = 3352570, upload-time = "2025-07-31T07:54:19.204Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/77/a9/3d7aa83955617cdf02f94e50aab5c830d205cfa4320cf124ff64acce3a8e/mypy-1.17.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3fbe6d5555bf608c47203baa3e72dbc6ec9965b3d7c318aa9a4ca76f465bd972", size = 11003299, upload-time = "2025-07-31T07:54:06.425Z" }, - { url = "https://files.pythonhosted.org/packages/83/e8/72e62ff837dd5caaac2b4a5c07ce769c8e808a00a65e5d8f94ea9c6f20ab/mypy-1.17.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:80ef5c058b7bce08c83cac668158cb7edea692e458d21098c7d3bce35a5d43e7", size = 10125451, upload-time = "2025-07-31T07:53:52.974Z" }, - { url = "https://files.pythonhosted.org/packages/7d/10/f3f3543f6448db11881776f26a0ed079865926b0c841818ee22de2c6bbab/mypy-1.17.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c4a580f8a70c69e4a75587bd925d298434057fe2a428faaf927ffe6e4b9a98df", size = 11916211, upload-time = "2025-07-31T07:53:18.879Z" }, - { url = "https://files.pythonhosted.org/packages/06/bf/63e83ed551282d67bb3f7fea2cd5561b08d2bb6eb287c096539feb5ddbc5/mypy-1.17.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dd86bb649299f09d987a2eebb4d52d10603224500792e1bee18303bbcc1ce390", size = 12652687, upload-time = "2025-07-31T07:53:30.544Z" }, - { url = "https://files.pythonhosted.org/packages/69/66/68f2eeef11facf597143e85b694a161868b3b006a5fbad50e09ea117ef24/mypy-1.17.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:a76906f26bd8d51ea9504966a9c25419f2e668f012e0bdf3da4ea1526c534d94", size = 12896322, upload-time = "2025-07-31T07:53:50.74Z" }, - { url = "https://files.pythonhosted.org/packages/a3/87/8e3e9c2c8bd0d7e071a89c71be28ad088aaecbadf0454f46a540bda7bca6/mypy-1.17.1-cp310-cp310-win_amd64.whl", hash = "sha256:e79311f2d904ccb59787477b7bd5d26f3347789c06fcd7656fa500875290264b", size = 9507962, upload-time = "2025-07-31T07:53:08.431Z" }, - { url = "https://files.pythonhosted.org/packages/46/cf/eadc80c4e0a70db1c08921dcc220357ba8ab2faecb4392e3cebeb10edbfa/mypy-1.17.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ad37544be07c5d7fba814eb370e006df58fed8ad1ef33ed1649cb1889ba6ff58", size = 10921009, upload-time = "2025-07-31T07:53:23.037Z" }, - { url = "https://files.pythonhosted.org/packages/5d/c1/c869d8c067829ad30d9bdae051046561552516cfb3a14f7f0347b7d973ee/mypy-1.17.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:064e2ff508e5464b4bd807a7c1625bc5047c5022b85c70f030680e18f37273a5", size = 10047482, upload-time = "2025-07-31T07:53:26.151Z" }, - { url = "https://files.pythonhosted.org/packages/98/b9/803672bab3fe03cee2e14786ca056efda4bb511ea02dadcedde6176d06d0/mypy-1.17.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:70401bbabd2fa1aa7c43bb358f54037baf0586f41e83b0ae67dd0534fc64edfd", size = 11832883, upload-time = "2025-07-31T07:53:47.948Z" }, - { url = "https://files.pythonhosted.org/packages/88/fb/fcdac695beca66800918c18697b48833a9a6701de288452b6715a98cfee1/mypy-1.17.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e92bdc656b7757c438660f775f872a669b8ff374edc4d18277d86b63edba6b8b", size = 12566215, upload-time = "2025-07-31T07:54:04.031Z" }, - { url = "https://files.pythonhosted.org/packages/7f/37/a932da3d3dace99ee8eb2043b6ab03b6768c36eb29a02f98f46c18c0da0e/mypy-1.17.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c1fdf4abb29ed1cb091cf432979e162c208a5ac676ce35010373ff29247bcad5", size = 12751956, upload-time = "2025-07-31T07:53:36.263Z" }, - { url = "https://files.pythonhosted.org/packages/8c/cf/6438a429e0f2f5cab8bc83e53dbebfa666476f40ee322e13cac5e64b79e7/mypy-1.17.1-cp311-cp311-win_amd64.whl", hash = "sha256:ff2933428516ab63f961644bc49bc4cbe42bbffb2cd3b71cc7277c07d16b1a8b", size = 9507307, upload-time = "2025-07-31T07:53:59.734Z" }, - { url = "https://files.pythonhosted.org/packages/17/a2/7034d0d61af8098ec47902108553122baa0f438df8a713be860f7407c9e6/mypy-1.17.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:69e83ea6553a3ba79c08c6e15dbd9bfa912ec1e493bf75489ef93beb65209aeb", size = 11086295, upload-time = "2025-07-31T07:53:28.124Z" }, - { url = "https://files.pythonhosted.org/packages/14/1f/19e7e44b594d4b12f6ba8064dbe136505cec813549ca3e5191e40b1d3cc2/mypy-1.17.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1b16708a66d38abb1e6b5702f5c2c87e133289da36f6a1d15f6a5221085c6403", size = 10112355, upload-time = "2025-07-31T07:53:21.121Z" }, - { url = "https://files.pythonhosted.org/packages/5b/69/baa33927e29e6b4c55d798a9d44db5d394072eef2bdc18c3e2048c9ed1e9/mypy-1.17.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:89e972c0035e9e05823907ad5398c5a73b9f47a002b22359b177d40bdaee7056", size = 11875285, upload-time = "2025-07-31T07:53:55.293Z" }, - { url = "https://files.pythonhosted.org/packages/90/13/f3a89c76b0a41e19490b01e7069713a30949d9a6c147289ee1521bcea245/mypy-1.17.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:03b6d0ed2b188e35ee6d5c36b5580cffd6da23319991c49ab5556c023ccf1341", size = 12737895, upload-time = "2025-07-31T07:53:43.623Z" }, - { url = "https://files.pythonhosted.org/packages/23/a1/c4ee79ac484241301564072e6476c5a5be2590bc2e7bfd28220033d2ef8f/mypy-1.17.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c837b896b37cd103570d776bda106eabb8737aa6dd4f248451aecf53030cdbeb", size = 12931025, upload-time = "2025-07-31T07:54:17.125Z" }, - { url = "https://files.pythonhosted.org/packages/89/b8/7409477be7919a0608900e6320b155c72caab4fef46427c5cc75f85edadd/mypy-1.17.1-cp312-cp312-win_amd64.whl", hash = "sha256:665afab0963a4b39dff7c1fa563cc8b11ecff7910206db4b2e64dd1ba25aed19", size = 9584664, upload-time = "2025-07-31T07:54:12.842Z" }, - { url = "https://files.pythonhosted.org/packages/5b/82/aec2fc9b9b149f372850291827537a508d6c4d3664b1750a324b91f71355/mypy-1.17.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:93378d3203a5c0800c6b6d850ad2f19f7a3cdf1a3701d3416dbf128805c6a6a7", size = 11075338, upload-time = "2025-07-31T07:53:38.873Z" }, - { url = "https://files.pythonhosted.org/packages/07/ac/ee93fbde9d2242657128af8c86f5d917cd2887584cf948a8e3663d0cd737/mypy-1.17.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:15d54056f7fe7a826d897789f53dd6377ec2ea8ba6f776dc83c2902b899fee81", size = 10113066, upload-time = "2025-07-31T07:54:14.707Z" }, - { url = "https://files.pythonhosted.org/packages/5a/68/946a1e0be93f17f7caa56c45844ec691ca153ee8b62f21eddda336a2d203/mypy-1.17.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:209a58fed9987eccc20f2ca94afe7257a8f46eb5df1fb69958650973230f91e6", size = 11875473, upload-time = "2025-07-31T07:53:14.504Z" }, - { url = "https://files.pythonhosted.org/packages/9f/0f/478b4dce1cb4f43cf0f0d00fba3030b21ca04a01b74d1cd272a528cf446f/mypy-1.17.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:099b9a5da47de9e2cb5165e581f158e854d9e19d2e96b6698c0d64de911dd849", size = 12744296, upload-time = "2025-07-31T07:53:03.896Z" }, - { url = "https://files.pythonhosted.org/packages/ca/70/afa5850176379d1b303f992a828de95fc14487429a7139a4e0bdd17a8279/mypy-1.17.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:fa6ffadfbe6994d724c5a1bb6123a7d27dd68fc9c059561cd33b664a79578e14", size = 12914657, upload-time = "2025-07-31T07:54:08.576Z" }, - { url = "https://files.pythonhosted.org/packages/53/f9/4a83e1c856a3d9c8f6edaa4749a4864ee98486e9b9dbfbc93842891029c2/mypy-1.17.1-cp313-cp313-win_amd64.whl", hash = "sha256:9a2b7d9180aed171f033c9f2fc6c204c1245cf60b0cb61cf2e7acc24eea78e0a", size = 9593320, upload-time = "2025-07-31T07:53:01.341Z" }, - { url = "https://files.pythonhosted.org/packages/38/56/79c2fac86da57c7d8c48622a05873eaab40b905096c33597462713f5af90/mypy-1.17.1-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:15a83369400454c41ed3a118e0cc58bd8123921a602f385cb6d6ea5df050c733", size = 11040037, upload-time = "2025-07-31T07:54:10.942Z" }, - { url = "https://files.pythonhosted.org/packages/4d/c3/adabe6ff53638e3cad19e3547268482408323b1e68bf082c9119000cd049/mypy-1.17.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:55b918670f692fc9fba55c3298d8a3beae295c5cded0a55dccdc5bbead814acd", size = 10131550, upload-time = "2025-07-31T07:53:41.307Z" }, - { url = "https://files.pythonhosted.org/packages/b8/c5/2e234c22c3bdeb23a7817af57a58865a39753bde52c74e2c661ee0cfc640/mypy-1.17.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:62761474061feef6f720149d7ba876122007ddc64adff5ba6f374fda35a018a0", size = 11872963, upload-time = "2025-07-31T07:53:16.878Z" }, - { url = "https://files.pythonhosted.org/packages/ab/26/c13c130f35ca8caa5f2ceab68a247775648fdcd6c9a18f158825f2bc2410/mypy-1.17.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c49562d3d908fd49ed0938e5423daed8d407774a479b595b143a3d7f87cdae6a", size = 12710189, upload-time = "2025-07-31T07:54:01.962Z" }, - { url = "https://files.pythonhosted.org/packages/82/df/c7d79d09f6de8383fe800521d066d877e54d30b4fb94281c262be2df84ef/mypy-1.17.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:397fba5d7616a5bc60b45c7ed204717eaddc38f826e3645402c426057ead9a91", size = 12900322, upload-time = "2025-07-31T07:53:10.551Z" }, - { url = "https://files.pythonhosted.org/packages/b8/98/3d5a48978b4f708c55ae832619addc66d677f6dc59f3ebad71bae8285ca6/mypy-1.17.1-cp314-cp314-win_amd64.whl", hash = "sha256:9d6b20b97d373f41617bd0708fd46aa656059af57f2ef72aa8c7d6a2b73b74ed", size = 9751879, upload-time = "2025-07-31T07:52:56.683Z" }, - { url = "https://files.pythonhosted.org/packages/29/cb/673e3d34e5d8de60b3a61f44f80150a738bff568cd6b7efb55742a605e98/mypy-1.17.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5d1092694f166a7e56c805caaf794e0585cabdbf1df36911c414e4e9abb62ae9", size = 10992466, upload-time = "2025-07-31T07:53:57.574Z" }, - { url = "https://files.pythonhosted.org/packages/0c/d0/fe1895836eea3a33ab801561987a10569df92f2d3d4715abf2cfeaa29cb2/mypy-1.17.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:79d44f9bfb004941ebb0abe8eff6504223a9c1ac51ef967d1263c6572bbebc99", size = 10117638, upload-time = "2025-07-31T07:53:34.256Z" }, - { url = "https://files.pythonhosted.org/packages/97/f3/514aa5532303aafb95b9ca400a31054a2bd9489de166558c2baaeea9c522/mypy-1.17.1-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b01586eed696ec905e61bd2568f48740f7ac4a45b3a468e6423a03d3788a51a8", size = 11915673, upload-time = "2025-07-31T07:52:59.361Z" }, - { url = "https://files.pythonhosted.org/packages/ab/c3/c0805f0edec96fe8e2c048b03769a6291523d509be8ee7f56ae922fa3882/mypy-1.17.1-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:43808d9476c36b927fbcd0b0255ce75efe1b68a080154a38ae68a7e62de8f0f8", size = 12649022, upload-time = "2025-07-31T07:53:45.92Z" }, - { url = "https://files.pythonhosted.org/packages/45/3e/d646b5a298ada21a8512fa7e5531f664535a495efa672601702398cea2b4/mypy-1.17.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:feb8cc32d319edd5859da2cc084493b3e2ce5e49a946377663cc90f6c15fb259", size = 12895536, upload-time = "2025-07-31T07:53:06.17Z" }, - { url = "https://files.pythonhosted.org/packages/14/55/e13d0dcd276975927d1f4e9e2ec4fd409e199f01bdc671717e673cc63a22/mypy-1.17.1-cp39-cp39-win_amd64.whl", hash = "sha256:d7598cf74c3e16539d4e2f0b8d8c318e00041553d83d4861f87c7a72e95ac24d", size = 9512564, upload-time = "2025-07-31T07:53:12.346Z" }, - { url = "https://files.pythonhosted.org/packages/1d/f3/8fcd2af0f5b806f6cf463efaffd3c9548a28f84220493ecd38d127b6b66d/mypy-1.17.1-py3-none-any.whl", hash = "sha256:a9f52c0351c21fe24c21d8c0eb1f62967b262d6729393397b6f443c3b773c3b9", size = 2283411, upload-time = "2025-07-31T07:53:24.664Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/b9/eb/2c92d8ea1e684440f54fa49ac5d9a5f19967b7b472a281f419e69a8d228e/mypy-1.14.1.tar.gz", hash = "sha256:7ec88144fe9b510e8475ec2f5f251992690fcf89ccb4500b214b4226abcd32d6", size = 3216051, upload-time = "2024-12-30T16:39:07.335Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9b/7a/87ae2adb31d68402da6da1e5f30c07ea6063e9f09b5e7cfc9dfa44075e74/mypy-1.14.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:52686e37cf13d559f668aa398dd7ddf1f92c5d613e4f8cb262be2fb4fedb0fcb", size = 11211002, upload-time = "2024-12-30T16:37:22.435Z" }, + { url = "https://files.pythonhosted.org/packages/e1/23/eada4c38608b444618a132be0d199b280049ded278b24cbb9d3fc59658e4/mypy-1.14.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:1fb545ca340537d4b45d3eecdb3def05e913299ca72c290326be19b3804b39c0", size = 10358400, upload-time = "2024-12-30T16:37:53.526Z" }, + { url = "https://files.pythonhosted.org/packages/43/c9/d6785c6f66241c62fd2992b05057f404237deaad1566545e9f144ced07f5/mypy-1.14.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:90716d8b2d1f4cd503309788e51366f07c56635a3309b0f6a32547eaaa36a64d", size = 12095172, upload-time = "2024-12-30T16:37:50.332Z" }, + { url = "https://files.pythonhosted.org/packages/c3/62/daa7e787770c83c52ce2aaf1a111eae5893de9e004743f51bfcad9e487ec/mypy-1.14.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2ae753f5c9fef278bcf12e1a564351764f2a6da579d4a81347e1d5a15819997b", size = 12828732, upload-time = "2024-12-30T16:37:29.96Z" }, + { url = "https://files.pythonhosted.org/packages/1b/a2/5fb18318a3637f29f16f4e41340b795da14f4751ef4f51c99ff39ab62e52/mypy-1.14.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:e0fe0f5feaafcb04505bcf439e991c6d8f1bf8b15f12b05feeed96e9e7bf1427", size = 13012197, upload-time = "2024-12-30T16:38:05.037Z" }, + { url = "https://files.pythonhosted.org/packages/28/99/e153ce39105d164b5f02c06c35c7ba958aaff50a2babba7d080988b03fe7/mypy-1.14.1-cp310-cp310-win_amd64.whl", hash = "sha256:7d54bd85b925e501c555a3227f3ec0cfc54ee8b6930bd6141ec872d1c572f81f", size = 9780836, upload-time = "2024-12-30T16:37:19.726Z" }, + { url = "https://files.pythonhosted.org/packages/da/11/a9422850fd506edbcdc7f6090682ecceaf1f87b9dd847f9df79942da8506/mypy-1.14.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f995e511de847791c3b11ed90084a7a0aafdc074ab88c5a9711622fe4751138c", size = 11120432, upload-time = "2024-12-30T16:37:11.533Z" }, + { url = "https://files.pythonhosted.org/packages/b6/9e/47e450fd39078d9c02d620545b2cb37993a8a8bdf7db3652ace2f80521ca/mypy-1.14.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d64169ec3b8461311f8ce2fd2eb5d33e2d0f2c7b49116259c51d0d96edee48d1", size = 10279515, upload-time = "2024-12-30T16:37:40.724Z" }, + { url = "https://files.pythonhosted.org/packages/01/b5/6c8d33bd0f851a7692a8bfe4ee75eb82b6983a3cf39e5e32a5d2a723f0c1/mypy-1.14.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ba24549de7b89b6381b91fbc068d798192b1b5201987070319889e93038967a8", size = 12025791, upload-time = "2024-12-30T16:36:58.73Z" }, + { url = "https://files.pythonhosted.org/packages/f0/4c/e10e2c46ea37cab5c471d0ddaaa9a434dc1d28650078ac1b56c2d7b9b2e4/mypy-1.14.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:183cf0a45457d28ff9d758730cd0210419ac27d4d3f285beda038c9083363b1f", size = 12749203, upload-time = "2024-12-30T16:37:03.741Z" }, + { url = "https://files.pythonhosted.org/packages/88/55/beacb0c69beab2153a0f57671ec07861d27d735a0faff135a494cd4f5020/mypy-1.14.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f2a0ecc86378f45347f586e4163d1769dd81c5a223d577fe351f26b179e148b1", size = 12885900, upload-time = "2024-12-30T16:37:57.948Z" }, + { url = "https://files.pythonhosted.org/packages/a2/75/8c93ff7f315c4d086a2dfcde02f713004357d70a163eddb6c56a6a5eff40/mypy-1.14.1-cp311-cp311-win_amd64.whl", hash = "sha256:ad3301ebebec9e8ee7135d8e3109ca76c23752bac1e717bc84cd3836b4bf3eae", size = 9777869, upload-time = "2024-12-30T16:37:33.428Z" }, + { url = "https://files.pythonhosted.org/packages/43/1b/b38c079609bb4627905b74fc6a49849835acf68547ac33d8ceb707de5f52/mypy-1.14.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:30ff5ef8519bbc2e18b3b54521ec319513a26f1bba19a7582e7b1f58a6e69f14", size = 11266668, upload-time = "2024-12-30T16:38:02.211Z" }, + { url = "https://files.pythonhosted.org/packages/6b/75/2ed0d2964c1ffc9971c729f7a544e9cd34b2cdabbe2d11afd148d7838aa2/mypy-1.14.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:cb9f255c18052343c70234907e2e532bc7e55a62565d64536dbc7706a20b78b9", size = 10254060, upload-time = "2024-12-30T16:37:46.131Z" }, + { url = "https://files.pythonhosted.org/packages/a1/5f/7b8051552d4da3c51bbe8fcafffd76a6823779101a2b198d80886cd8f08e/mypy-1.14.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8b4e3413e0bddea671012b063e27591b953d653209e7a4fa5e48759cda77ca11", size = 11933167, upload-time = "2024-12-30T16:37:43.534Z" }, + { url = "https://files.pythonhosted.org/packages/04/90/f53971d3ac39d8b68bbaab9a4c6c58c8caa4d5fd3d587d16f5927eeeabe1/mypy-1.14.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:553c293b1fbdebb6c3c4030589dab9fafb6dfa768995a453d8a5d3b23784af2e", size = 12864341, upload-time = "2024-12-30T16:37:36.249Z" }, + { url = "https://files.pythonhosted.org/packages/03/d2/8bc0aeaaf2e88c977db41583559319f1821c069e943ada2701e86d0430b7/mypy-1.14.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fad79bfe3b65fe6a1efaed97b445c3d37f7be9fdc348bdb2d7cac75579607c89", size = 12972991, upload-time = "2024-12-30T16:37:06.743Z" }, + { url = "https://files.pythonhosted.org/packages/6f/17/07815114b903b49b0f2cf7499f1c130e5aa459411596668267535fe9243c/mypy-1.14.1-cp312-cp312-win_amd64.whl", hash = "sha256:8fa2220e54d2946e94ab6dbb3ba0a992795bd68b16dc852db33028df2b00191b", size = 9879016, upload-time = "2024-12-30T16:37:15.02Z" }, + { url = "https://files.pythonhosted.org/packages/9e/15/bb6a686901f59222275ab228453de741185f9d54fecbaacec041679496c6/mypy-1.14.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:92c3ed5afb06c3a8e188cb5da4984cab9ec9a77ba956ee419c68a388b4595255", size = 11252097, upload-time = "2024-12-30T16:37:25.144Z" }, + { url = "https://files.pythonhosted.org/packages/f8/b3/8b0f74dfd072c802b7fa368829defdf3ee1566ba74c32a2cb2403f68024c/mypy-1.14.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:dbec574648b3e25f43d23577309b16534431db4ddc09fda50841f1e34e64ed34", size = 10239728, upload-time = "2024-12-30T16:38:08.634Z" }, + { url = "https://files.pythonhosted.org/packages/c5/9b/4fd95ab20c52bb5b8c03cc49169be5905d931de17edfe4d9d2986800b52e/mypy-1.14.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8c6d94b16d62eb3e947281aa7347d78236688e21081f11de976376cf010eb31a", size = 11924965, upload-time = "2024-12-30T16:38:12.132Z" }, + { url = "https://files.pythonhosted.org/packages/56/9d/4a236b9c57f5d8f08ed346914b3f091a62dd7e19336b2b2a0d85485f82ff/mypy-1.14.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d4b19b03fdf54f3c5b2fa474c56b4c13c9dbfb9a2db4370ede7ec11a2c5927d9", size = 12867660, upload-time = "2024-12-30T16:38:17.342Z" }, + { url = "https://files.pythonhosted.org/packages/40/88/a61a5497e2f68d9027de2bb139c7bb9abaeb1be1584649fa9d807f80a338/mypy-1.14.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:0c911fde686394753fff899c409fd4e16e9b294c24bfd5e1ea4675deae1ac6fd", size = 12969198, upload-time = "2024-12-30T16:38:32.839Z" }, + { url = "https://files.pythonhosted.org/packages/54/da/3d6fc5d92d324701b0c23fb413c853892bfe0e1dbe06c9138037d459756b/mypy-1.14.1-cp313-cp313-win_amd64.whl", hash = "sha256:8b21525cb51671219f5307be85f7e646a153e5acc656e5cebf64bfa076c50107", size = 9885276, upload-time = "2024-12-30T16:38:20.828Z" }, + { url = "https://files.pythonhosted.org/packages/ca/1f/186d133ae2514633f8558e78cd658070ba686c0e9275c5a5c24a1e1f0d67/mypy-1.14.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:3888a1816d69f7ab92092f785a462944b3ca16d7c470d564165fe703b0970c35", size = 11200493, upload-time = "2024-12-30T16:38:26.935Z" }, + { url = "https://files.pythonhosted.org/packages/af/fc/4842485d034e38a4646cccd1369f6b1ccd7bc86989c52770d75d719a9941/mypy-1.14.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:46c756a444117c43ee984bd055db99e498bc613a70bbbc120272bd13ca579fbc", size = 10357702, upload-time = "2024-12-30T16:38:50.623Z" }, + { url = "https://files.pythonhosted.org/packages/b4/e6/457b83f2d701e23869cfec013a48a12638f75b9d37612a9ddf99072c1051/mypy-1.14.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:27fc248022907e72abfd8e22ab1f10e903915ff69961174784a3900a8cba9ad9", size = 12091104, upload-time = "2024-12-30T16:38:53.735Z" }, + { url = "https://files.pythonhosted.org/packages/f1/bf/76a569158db678fee59f4fd30b8e7a0d75bcbaeef49edd882a0d63af6d66/mypy-1.14.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:499d6a72fb7e5de92218db961f1a66d5f11783f9ae549d214617edab5d4dbdbb", size = 12830167, upload-time = "2024-12-30T16:38:56.437Z" }, + { url = "https://files.pythonhosted.org/packages/43/bc/0bc6b694b3103de9fed61867f1c8bd33336b913d16831431e7cb48ef1c92/mypy-1.14.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:57961db9795eb566dc1d1b4e9139ebc4c6b0cb6e7254ecde69d1552bf7613f60", size = 13013834, upload-time = "2024-12-30T16:38:59.204Z" }, + { url = "https://files.pythonhosted.org/packages/b0/79/5f5ec47849b6df1e6943d5fd8e6632fbfc04b4fd4acfa5a5a9535d11b4e2/mypy-1.14.1-cp39-cp39-win_amd64.whl", hash = "sha256:07ba89fdcc9451f2ebb02853deb6aaaa3d2239a236669a63ab3801bbf923ef5c", size = 9781231, upload-time = "2024-12-30T16:39:05.124Z" }, + { url = "https://files.pythonhosted.org/packages/a0/b5/32dd67b69a16d088e533962e5044e51004176a9952419de0370cdaead0f8/mypy-1.14.1-py3-none-any.whl", hash = "sha256:b66a60cc4073aeb8ae00057f9c1f64d49e90f918fbcef9a977eb121da8b8f1d1", size = 2752905, upload-time = "2024-12-30T16:38:42.021Z" }, ] [[package]] @@ -1123,15 +1116,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484", size = 66469, upload-time = "2025-04-19T11:48:57.875Z" }, ] -[[package]] -name = "pathspec" -version = "0.12.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/ca/bc/f35b8446f4531a7cb215605d100cd88b7ac6f44ab3fc94870c120ab3adbf/pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712", size = 51043, upload-time = "2023-12-10T22:30:45Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/cc/20/ff623b09d963f88bfde16306a54e12ee5ea43e9b597108672ff3a408aad6/pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08", size = 31191, upload-time = "2023-12-10T22:30:43.14Z" }, -] - [[package]] name = "pip" version = "25.2" @@ -1331,7 +1315,7 @@ provides-extras = ["aws", "docs", "encryption", "gssapi", "ocsp", "snappy", "tes [package.metadata.requires-dev] coverage = [ - { name = "coverage", specifier = ">=5,<=7.10.3" }, + { name = "coverage", specifier = ">=5,<=7.5" }, { name = "pytest-cov" }, ] dev = [{ name = "pre-commit", specifier = ">=4.0" }] @@ -1345,9 +1329,9 @@ perf = [{ name = "simplejson" }] pip = [{ name = "pip" }] pymongocrypt-source = [{ name = "pymongocrypt", git = "https://github.com/mongodb/libmongocrypt?subdirectory=bindings%2Fpython&rev=master" }] typing = [ - { name = "mypy", specifier = "==1.17.1" }, + { name = "mypy", specifier = "==1.14.1" }, { name = "pip" }, - { name = "pyright", specifier = "==1.1.403" }, + { name = "pyright", specifier = "==1.1.392.post0" }, { name = "typing-extensions" }, ] @@ -1391,15 +1375,15 @@ wheels = [ [[package]] name = "pyright" -version = "1.1.403" +version = "1.1.392.post0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "nodeenv" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/fe/f6/35f885264ff08c960b23d1542038d8da86971c5d8c955cfab195a4f672d7/pyright-1.1.403.tar.gz", hash = "sha256:3ab69b9f41c67fb5bbb4d7a36243256f0d549ed3608678d381d5f51863921104", size = 3913526, upload-time = "2025-07-09T07:15:52.882Z" } +sdist = { url = "https://files.pythonhosted.org/packages/66/df/3c6f6b08fba7ccf49b114dfc4bb33e25c299883fd763f93fad47ef8bc58d/pyright-1.1.392.post0.tar.gz", hash = "sha256:3b7f88de74a28dcfa90c7d90c782b6569a48c2be5f9d4add38472bdaac247ebd", size = 3789911, upload-time = "2025-01-15T15:01:20.913Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/49/b6/b04e5c2f41a5ccad74a1a4759da41adb20b4bc9d59a5e08d29ba60084d07/pyright-1.1.403-py3-none-any.whl", hash = "sha256:c0eeca5aa76cbef3fcc271259bbd785753c7ad7bcac99a9162b4c4c7daed23b3", size = 5684504, upload-time = "2025-07-09T07:15:50.958Z" }, + { url = "https://files.pythonhosted.org/packages/e7/b1/a18de17f40e4f61ca58856b9ef9b0febf74ff88978c3f7776f910071f567/pyright-1.1.392.post0-py3-none-any.whl", hash = "sha256:252f84458a46fa2f0fd4e2f91fc74f50b9ca52c757062e93f6c250c0d8329eb2", size = 5595487, upload-time = "2025-01-15T15:01:17.775Z" }, ] [[package]] From 6f38a9bd0c4f23daad66c561122433deb43c7b71 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 2 Sep 2025 12:17:58 -0500 Subject: [PATCH 31/53] add test for limiting maxConnecting --- test/asynchronous/test_pooling.py | 86 +++++++++++++++++++------------ test/test_pooling.py | 84 ++++++++++++++++++------------ 2 files changed, 106 insertions(+), 64 deletions(-) diff --git a/test/asynchronous/test_pooling.py b/test/asynchronous/test_pooling.py index a26a93a29e..397c4cd1cf 100644 --- a/test/asynchronous/test_pooling.py +++ b/test/asynchronous/test_pooling.py @@ -394,11 +394,15 @@ async def test_checkout_more_than_max_pool_size(self): await asyncio.sleep(0.05) await pool.close() - async def test_maxConnecting(self): - client = await self.async_rs_or_single_client() - await self.client.test.test.insert_one({}) - self.addAsyncCleanup(self.client.test.test.delete_many, {}) + async def _check_maxConnecting( + self, client: AsyncMongoClient, backoff=False + ) -> tuple[int, int]: + await client.test.test.insert_one({}) + + self.addAsyncCleanup(client.test.test.delete_many, {}) pool = await async_get_pool(client) + if backoff: + pool._backoff = 1 docs = [] # Run 50 short running operations @@ -411,15 +415,21 @@ async def find_one(): for task in tasks: await task.join(10) - self.assertEqual(len(docs), 50) - self.assertLessEqual(len(pool.conns), 50) + return len(docs), len(pool.conns) + + async def test_maxConnecting(self): + client = await self.async_rs_or_single_client() + num_docs, num_conns = await self._check_maxConnecting(client) + + self.assertEqual(num_docs, 50) + self.assertLessEqual(num_conns, 50) # TLS and auth make connection establishment more expensive than # the query which leads to more threads hitting maxConnecting. # The end result is fewer total connections and better latency. if async_client_context.tls and async_client_context.auth_enabled: - self.assertLessEqual(len(pool.conns), 30) + self.assertLessEqual(num_conns, 30) else: - self.assertLessEqual(len(pool.conns), 50) + self.assertLessEqual(num_conns, 50) # MongoDB 4.4.1 with auth + ssl: # maxConnecting = 2: 6 connections in ~0.231+ seconds # maxConnecting = unbounded: 50 connections in ~0.642+ seconds @@ -427,7 +437,7 @@ async def find_one(): # MongoDB 4.4.1 with no-auth no-ssl Python 3.8: # maxConnecting = 2: 15-22 connections in ~0.108+ seconds # maxConnecting = unbounded: 30+ connections in ~0.140+ seconds - print(len(pool.conns)) + print(num_conns) @async_client_context.require_failCommand_appName async def test_csot_timeout_message(self): @@ -513,8 +523,33 @@ async def test_connection_timeout_message(self): str(error.exception), ) + async def test_pool_check_backoff(self): + # Test that Pool recovers from two connection failures in a row. + # This exercises code at the end of Pool._check(). + cx_pool = await self.create_pool(max_pool_size=1, connect_timeout=1, wait_queue_timeout=1) + self.addAsyncCleanup(cx_pool.close) + + async with cx_pool.checkout() as conn: + # Simulate a closed socket without telling the Connection it's + # closed. + await conn.conn.close() + + # Enable backoff. + cx_pool._backoff = 1 + + # Swap pool's address with a bad one. + address, cx_pool.address = cx_pool.address, ("foo.com", 1234) + with self.assertRaises(AutoReconnect): + async with cx_pool.checkout(): + pass + + # Back to normal, semaphore was correctly released. + cx_pool.address = address + async with cx_pool.checkout(): + pass + @async_client_context.require_failCommand_appName - async def test_pool_backoff_preserves_existing_collections(self): + async def test_pool_backoff_preserves_existing_connections(self): client = await self.async_rs_or_single_client() coll = self.db.t pool = await async_get_pool(client) @@ -549,30 +584,17 @@ async def test_pool_backoff_preserves_existing_collections(self): await t.join() await pool.close() - async def test_pool_check_backoff(self): - # Test that Pool recovers from two connection failures in a row. - # This exercises code at the end of Pool._check(). - cx_pool = await self.create_pool(max_pool_size=1, connect_timeout=1, wait_queue_timeout=1) - self.addAsyncCleanup(cx_pool.close) - - async with cx_pool.checkout() as conn: - # Simulate a closed socket without telling the Connection it's - # closed. - await conn.conn.close() - - # Enable backoff. - cx_pool._backoff = 1 + async def test_pool_backoff_limits_maxConnecting(self): + client = await self.async_rs_or_single_client() + _, baseline_conns = await self._check_maxConnecting(client) + client.close() - # Swap pool's address with a bad one. - address, cx_pool.address = cx_pool.address, ("foo.com", 1234) - with self.assertRaises(AutoReconnect): - async with cx_pool.checkout(): - pass + client = await self.async_rs_or_single_client() + _, backoff_conns = await self._check_maxConnecting(client, backoff=True) + client.close() - # Back to normal, semaphore was correctly released. - cx_pool.address = address - async with cx_pool.checkout(): - pass + # We should have created less conns due to limiting maxConnecting. + self.assertLess(backoff_conns, baseline_conns) class TestPoolMaxSize(_TestPoolingBase): diff --git a/test/test_pooling.py b/test/test_pooling.py index 3f76814143..0adeccd095 100644 --- a/test/test_pooling.py +++ b/test/test_pooling.py @@ -394,11 +394,13 @@ def test_checkout_more_than_max_pool_size(self): time.sleep(0.05) pool.close() - def test_maxConnecting(self): - client = self.rs_or_single_client() - self.client.test.test.insert_one({}) - self.addCleanup(self.client.test.test.delete_many, {}) + def _check_maxConnecting(self, client: MongoClient, backoff=False) -> tuple[int, int]: + client.test.test.insert_one({}) + + self.addCleanup(client.test.test.delete_many, {}) pool = get_pool(client) + if backoff: + pool._backoff = 1 docs = [] # Run 50 short running operations @@ -411,15 +413,21 @@ def find_one(): for task in tasks: task.join(10) - self.assertEqual(len(docs), 50) - self.assertLessEqual(len(pool.conns), 50) + return len(docs), len(pool.conns) + + def test_maxConnecting(self): + client = self.rs_or_single_client() + num_docs, num_conns = self._check_maxConnecting(client) + + self.assertEqual(num_docs, 50) + self.assertLessEqual(num_conns, 50) # TLS and auth make connection establishment more expensive than # the query which leads to more threads hitting maxConnecting. # The end result is fewer total connections and better latency. if client_context.tls and client_context.auth_enabled: - self.assertLessEqual(len(pool.conns), 30) + self.assertLessEqual(num_conns, 30) else: - self.assertLessEqual(len(pool.conns), 50) + self.assertLessEqual(num_conns, 50) # MongoDB 4.4.1 with auth + ssl: # maxConnecting = 2: 6 connections in ~0.231+ seconds # maxConnecting = unbounded: 50 connections in ~0.642+ seconds @@ -427,7 +435,7 @@ def find_one(): # MongoDB 4.4.1 with no-auth no-ssl Python 3.8: # maxConnecting = 2: 15-22 connections in ~0.108+ seconds # maxConnecting = unbounded: 30+ connections in ~0.140+ seconds - print(len(pool.conns)) + print(num_conns) @client_context.require_failCommand_appName def test_csot_timeout_message(self): @@ -511,8 +519,33 @@ def test_connection_timeout_message(self): str(error.exception), ) + def test_pool_check_backoff(self): + # Test that Pool recovers from two connection failures in a row. + # This exercises code at the end of Pool._check(). + cx_pool = self.create_pool(max_pool_size=1, connect_timeout=1, wait_queue_timeout=1) + self.addCleanup(cx_pool.close) + + with cx_pool.checkout() as conn: + # Simulate a closed socket without telling the Connection it's + # closed. + conn.conn.close() + + # Enable backoff. + cx_pool._backoff = 1 + + # Swap pool's address with a bad one. + address, cx_pool.address = cx_pool.address, ("foo.com", 1234) + with self.assertRaises(AutoReconnect): + with cx_pool.checkout(): + pass + + # Back to normal, semaphore was correctly released. + cx_pool.address = address + with cx_pool.checkout(): + pass + @client_context.require_failCommand_appName - def test_pool_backoff_preserves_existing_collections(self): + def test_pool_backoff_preserves_existing_connections(self): client = self.rs_or_single_client() coll = self.db.t pool = get_pool(client) @@ -547,30 +580,17 @@ def test_pool_backoff_preserves_existing_collections(self): t.join() pool.close() - def test_pool_check_backoff(self): - # Test that Pool recovers from two connection failures in a row. - # This exercises code at the end of Pool._check(). - cx_pool = self.create_pool(max_pool_size=1, connect_timeout=1, wait_queue_timeout=1) - self.addCleanup(cx_pool.close) - - with cx_pool.checkout() as conn: - # Simulate a closed socket without telling the Connection it's - # closed. - conn.conn.close() - - # Enable backoff. - cx_pool._backoff = 1 + def test_pool_backoff_limits_maxConnecting(self): + client = self.rs_or_single_client() + _, baseline_conns = self._check_maxConnecting(client) + client.close() - # Swap pool's address with a bad one. - address, cx_pool.address = cx_pool.address, ("foo.com", 1234) - with self.assertRaises(AutoReconnect): - with cx_pool.checkout(): - pass + client = self.rs_or_single_client() + _, backoff_conns = self._check_maxConnecting(client, backoff=True) + client.close() - # Back to normal, semaphore was correctly released. - cx_pool.address = address - with cx_pool.checkout(): - pass + # We should have created less conns due to limiting maxConnecting. + self.assertLess(backoff_conns, baseline_conns) class TestPoolMaxSize(_TestPoolingBase): From 09972487dce87bf5817666c6a07899a7ce662869 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 3 Sep 2025 16:34:21 -0500 Subject: [PATCH 32/53] add sdam check --- pymongo/asynchronous/pool.py | 2 +- pymongo/synchronous/pool.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pymongo/asynchronous/pool.py b/pymongo/asynchronous/pool.py index 8075eb466e..6d0837f3d0 100644 --- a/pymongo/asynchronous/pool.py +++ b/pymongo/asynchronous/pool.py @@ -1106,7 +1106,7 @@ async def connect(self, handler: Optional[_MongoClientErrorHandler] = None) -> A async with self.lock: self.active_contexts.discard(conn.cancel_context) # Enter backoff mode and reconnect on establishment failure. - if type(e) == AutoReconnect: + if not self.is_sdam and type(e) == AutoReconnect: await conn.close_conn(ConnectionClosedReason.ERROR) self._backoff += 1 # TODO: emit a message about backoff. diff --git a/pymongo/synchronous/pool.py b/pymongo/synchronous/pool.py index 78125ef042..6ada3eb066 100644 --- a/pymongo/synchronous/pool.py +++ b/pymongo/synchronous/pool.py @@ -1102,7 +1102,7 @@ def connect(self, handler: Optional[_MongoClientErrorHandler] = None) -> Connect with self.lock: self.active_contexts.discard(conn.cancel_context) # Enter backoff mode and reconnect on establishment failure. - if type(e) == AutoReconnect: + if not self.is_sdam and type(e) == AutoReconnect: conn.close_conn(ConnectionClosedReason.ERROR) self._backoff += 1 # TODO: emit a message about backoff. From 320cb5421dcba4c59048ce045788308fb4b7a1d8 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 4 Sep 2025 15:12:29 -0500 Subject: [PATCH 33/53] fix pool backoff --- pymongo/asynchronous/pool.py | 9 ++++----- pymongo/synchronous/pool.py | 9 ++++----- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/pymongo/asynchronous/pool.py b/pymongo/asynchronous/pool.py index 6d0837f3d0..d9db6c0c8a 100644 --- a/pymongo/asynchronous/pool.py +++ b/pymongo/asynchronous/pool.py @@ -1105,13 +1105,12 @@ async def connect(self, handler: Optional[_MongoClientErrorHandler] = None) -> A except BaseException as e: async with self.lock: self.active_contexts.discard(conn.cancel_context) - # Enter backoff mode and reconnect on establishment failure. + # Handle system overload condition. When the base AutoReconnect is + # raised and we are not an sdam pool, add to backoff and add the + # appropriate error label. if not self.is_sdam and type(e) == AutoReconnect: - await conn.close_conn(ConnectionClosedReason.ERROR) self._backoff += 1 - # TODO: emit a message about backoff. - print("backing off", self._backoff) # noqa: T201 - return await self.connect(handler) + e._add_error_label("SystemOverloaded") await conn.close_conn(ConnectionClosedReason.ERROR) raise diff --git a/pymongo/synchronous/pool.py b/pymongo/synchronous/pool.py index 6ada3eb066..abfa44fbf8 100644 --- a/pymongo/synchronous/pool.py +++ b/pymongo/synchronous/pool.py @@ -1101,13 +1101,12 @@ def connect(self, handler: Optional[_MongoClientErrorHandler] = None) -> Connect except BaseException as e: with self.lock: self.active_contexts.discard(conn.cancel_context) - # Enter backoff mode and reconnect on establishment failure. + # Handle system overload condition. When the base AutoReconnect is + # raised and we are not an sdam pool, add to backoff and add the + # appropriate error label. if not self.is_sdam and type(e) == AutoReconnect: - conn.close_conn(ConnectionClosedReason.ERROR) self._backoff += 1 - # TODO: emit a message about backoff. - print("backing off", self._backoff) # noqa: T201 - return self.connect(handler) + e._add_error_label("SystemOverloaded") conn.close_conn(ConnectionClosedReason.ERROR) raise From 7734af7089c25b57a448cc391fb7a1279f817735 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 4 Sep 2025 15:16:45 -0500 Subject: [PATCH 34/53] do not clear the pool on SystemOverloaded --- pymongo/asynchronous/topology.py | 4 +++- pymongo/synchronous/topology.py | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/pymongo/asynchronous/topology.py b/pymongo/asynchronous/topology.py index 283aabc690..e0e8821c43 100644 --- a/pymongo/asynchronous/topology.py +++ b/pymongo/asynchronous/topology.py @@ -890,7 +890,9 @@ async def _handle_error(self, address: _Address, err_ctx: _ErrorContext) -> None # Clear the pool. await server.reset(service_id) elif isinstance(error, ConnectionFailure): - if isinstance(error, WaitQueueTimeoutError): + if isinstance(error, WaitQueueTimeoutError) or error.has_error_label( + "SystemOverloaded" + ): return # "Client MUST replace the server's description with type Unknown # ... MUST NOT request an immediate check of the server." diff --git a/pymongo/synchronous/topology.py b/pymongo/synchronous/topology.py index a4ca0e6e0f..16e38eb3fa 100644 --- a/pymongo/synchronous/topology.py +++ b/pymongo/synchronous/topology.py @@ -888,7 +888,9 @@ def _handle_error(self, address: _Address, err_ctx: _ErrorContext) -> None: # Clear the pool. server.reset(service_id) elif isinstance(error, ConnectionFailure): - if isinstance(error, WaitQueueTimeoutError): + if isinstance(error, WaitQueueTimeoutError) or error.has_error_label( + "SystemOverloaded" + ): return # "Client MUST replace the server's description with type Unknown # ... MUST NOT request an immediate check of the server." From 452cdd60807070b2f9eab66fb9db311e0975a172 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 4 Sep 2025 15:33:59 -0500 Subject: [PATCH 35/53] add retryable label --- pymongo/asynchronous/pool.py | 2 ++ pymongo/synchronous/pool.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/pymongo/asynchronous/pool.py b/pymongo/asynchronous/pool.py index d9db6c0c8a..ec67899486 100644 --- a/pymongo/asynchronous/pool.py +++ b/pymongo/asynchronous/pool.py @@ -1111,6 +1111,8 @@ async def connect(self, handler: Optional[_MongoClientErrorHandler] = None) -> A if not self.is_sdam and type(e) == AutoReconnect: self._backoff += 1 e._add_error_label("SystemOverloaded") + e._add_error_label("Retryable") + print("Setting backoff:", self._backoff) # noqa: T201 await conn.close_conn(ConnectionClosedReason.ERROR) raise diff --git a/pymongo/synchronous/pool.py b/pymongo/synchronous/pool.py index abfa44fbf8..fdd12bb728 100644 --- a/pymongo/synchronous/pool.py +++ b/pymongo/synchronous/pool.py @@ -1107,6 +1107,8 @@ def connect(self, handler: Optional[_MongoClientErrorHandler] = None) -> Connect if not self.is_sdam and type(e) == AutoReconnect: self._backoff += 1 e._add_error_label("SystemOverloaded") + e._add_error_label("Retryable") + print("Setting backoff:", self._backoff) # noqa: T201 conn.close_conn(ConnectionClosedReason.ERROR) raise From 1f44c48d7ad1eeab1b0ce7bb0e4217a66379b853 Mon Sep 17 00:00:00 2001 From: Noah Stapp Date: Wed, 3 Sep 2025 14:18:51 -0400 Subject: [PATCH 36/53] PYTHON-5521 - Update TestBsonSizeBatches.test_06_insert_fails_over_16MiB error codes (#2515) (cherry picked from commit c0e0554a3b69b7c3140a792934682c19b43fc269) --- test/asynchronous/test_encryption.py | 2 +- test/test_encryption.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/asynchronous/test_encryption.py b/test/asynchronous/test_encryption.py index f6afa4b2a3..adba824143 100644 --- a/test/asynchronous/test_encryption.py +++ b/test/asynchronous/test_encryption.py @@ -1276,7 +1276,7 @@ async def test_06_insert_fails_over_16MiB(self): with self.assertRaises(BulkWriteError) as ctx: await self.coll_encrypted.bulk_write([InsertOne(doc)]) err = ctx.exception.details["writeErrors"][0] - self.assertEqual(2, err["code"]) + self.assertIn(err["code"], [2, 10334]) self.assertIn("object to insert too large", err["errmsg"]) diff --git a/test/test_encryption.py b/test/test_encryption.py index 5c8813203d..1a307f56ee 100644 --- a/test/test_encryption.py +++ b/test/test_encryption.py @@ -1272,7 +1272,7 @@ def test_06_insert_fails_over_16MiB(self): with self.assertRaises(BulkWriteError) as ctx: self.coll_encrypted.bulk_write([InsertOne(doc)]) err = ctx.exception.details["writeErrors"][0] - self.assertEqual(2, err["code"]) + self.assertIn(err["code"], [2, 10334]) self.assertIn("object to insert too large", err["errmsg"]) From fa5c1515113bcbe4e9b0ba0b094bd385cecaad0f Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 5 Sep 2025 06:15:23 -0500 Subject: [PATCH 37/53] fix test --- test/asynchronous/test_pooling.py | 3 ++- test/test_pooling.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/test/asynchronous/test_pooling.py b/test/asynchronous/test_pooling.py index 397c4cd1cf..37b1cd8b84 100644 --- a/test/asynchronous/test_pooling.py +++ b/test/asynchronous/test_pooling.py @@ -399,7 +399,6 @@ async def _check_maxConnecting( ) -> tuple[int, int]: await client.test.test.insert_one({}) - self.addAsyncCleanup(client.test.test.delete_many, {}) pool = await async_get_pool(client) if backoff: pool._backoff = 1 @@ -415,6 +414,8 @@ async def find_one(): for task in tasks: await task.join(10) + await client.test.test.delete_many({}) + return len(docs), len(pool.conns) async def test_maxConnecting(self): diff --git a/test/test_pooling.py b/test/test_pooling.py index 0adeccd095..f0d3140c66 100644 --- a/test/test_pooling.py +++ b/test/test_pooling.py @@ -397,7 +397,6 @@ def test_checkout_more_than_max_pool_size(self): def _check_maxConnecting(self, client: MongoClient, backoff=False) -> tuple[int, int]: client.test.test.insert_one({}) - self.addCleanup(client.test.test.delete_many, {}) pool = get_pool(client) if backoff: pool._backoff = 1 @@ -413,6 +412,8 @@ def find_one(): for task in tasks: task.join(10) + client.test.test.delete_many({}) + return len(docs), len(pool.conns) def test_maxConnecting(self): From 0ab78e43b7937972fe797102ac9bf13ea1cbd89a Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 5 Sep 2025 06:32:43 -0500 Subject: [PATCH 38/53] fix test --- test/asynchronous/test_pooling.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/asynchronous/test_pooling.py b/test/asynchronous/test_pooling.py index 37b1cd8b84..1d0028f43c 100644 --- a/test/asynchronous/test_pooling.py +++ b/test/asynchronous/test_pooling.py @@ -588,11 +588,11 @@ async def test_pool_backoff_preserves_existing_connections(self): async def test_pool_backoff_limits_maxConnecting(self): client = await self.async_rs_or_single_client() _, baseline_conns = await self._check_maxConnecting(client) - client.close() + await client.close() client = await self.async_rs_or_single_client() _, backoff_conns = await self._check_maxConnecting(client, backoff=True) - client.close() + await client.close() # We should have created less conns due to limiting maxConnecting. self.assertLess(backoff_conns, baseline_conns) From 07d023326dcbf5921df49ba3bc1781eac6077fec Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 5 Sep 2025 08:36:19 -0500 Subject: [PATCH 39/53] Revert "update handshake error tests" This reverts commit 532c1b8cba86e037f602b54c762e786ef9786fc4. --- .../unified/handshakeError.json | 96 +++++++++++++++++++ .../unified/handshakeError.json | 60 ++++++++++++ 2 files changed, 156 insertions(+) diff --git a/test/retryable_reads/unified/handshakeError.json b/test/retryable_reads/unified/handshakeError.json index 6a58084f6d..2921d8a954 100644 --- a/test/retryable_reads/unified/handshakeError.json +++ b/test/retryable_reads/unified/handshakeError.json @@ -116,6 +116,9 @@ { "connectionCheckOutStartedEvent": {} }, + { + "connectionCheckOutStartedEvent": {} + }, { "connectionCheckOutStartedEvent": {} } @@ -206,6 +209,9 @@ { "connectionCheckOutStartedEvent": {} }, + { + "connectionCheckOutStartedEvent": {} + }, { "connectionCheckOutStartedEvent": {} } @@ -293,6 +299,9 @@ { "connectionCheckOutStartedEvent": {} }, + { + "connectionCheckOutStartedEvent": {} + }, { "connectionCheckOutStartedEvent": {} } @@ -380,6 +389,9 @@ { "connectionCheckOutStartedEvent": {} }, + { + "connectionCheckOutStartedEvent": {} + }, { "connectionCheckOutStartedEvent": {} } @@ -476,6 +488,9 @@ { "connectionCheckOutStartedEvent": {} }, + { + "connectionCheckOutStartedEvent": {} + }, { "connectionCheckOutStartedEvent": {} } @@ -572,6 +587,9 @@ { "connectionCheckOutStartedEvent": {} }, + { + "connectionCheckOutStartedEvent": {} + }, { "connectionCheckOutStartedEvent": {} } @@ -674,6 +692,9 @@ { "connectionCheckOutStartedEvent": {} }, + { + "connectionCheckOutStartedEvent": {} + }, { "connectionCheckOutStartedEvent": {} } @@ -776,6 +797,9 @@ { "connectionCheckOutStartedEvent": {} }, + { + "connectionCheckOutStartedEvent": {} + }, { "connectionCheckOutStartedEvent": {} } @@ -866,6 +890,9 @@ { "connectionCheckOutStartedEvent": {} }, + { + "connectionCheckOutStartedEvent": {} + }, { "connectionCheckOutStartedEvent": {} } @@ -956,6 +983,9 @@ { "connectionCheckOutStartedEvent": {} }, + { + "connectionCheckOutStartedEvent": {} + }, { "connectionCheckOutStartedEvent": {} } @@ -1046,6 +1076,9 @@ { "connectionCheckOutStartedEvent": {} }, + { + "connectionCheckOutStartedEvent": {} + }, { "connectionCheckOutStartedEvent": {} } @@ -1136,6 +1169,9 @@ { "connectionCheckOutStartedEvent": {} }, + { + "connectionCheckOutStartedEvent": {} + }, { "connectionCheckOutStartedEvent": {} } @@ -1232,6 +1268,9 @@ { "connectionCheckOutStartedEvent": {} }, + { + "connectionCheckOutStartedEvent": {} + }, { "connectionCheckOutStartedEvent": {} } @@ -1328,6 +1367,9 @@ { "connectionCheckOutStartedEvent": {} }, + { + "connectionCheckOutStartedEvent": {} + }, { "connectionCheckOutStartedEvent": {} } @@ -1418,6 +1460,9 @@ { "connectionCheckOutStartedEvent": {} }, + { + "connectionCheckOutStartedEvent": {} + }, { "connectionCheckOutStartedEvent": {} } @@ -1508,6 +1553,9 @@ { "connectionCheckOutStartedEvent": {} }, + { + "connectionCheckOutStartedEvent": {} + }, { "connectionCheckOutStartedEvent": {} } @@ -1598,6 +1646,9 @@ { "connectionCheckOutStartedEvent": {} }, + { + "connectionCheckOutStartedEvent": {} + }, { "connectionCheckOutStartedEvent": {} } @@ -1688,6 +1739,9 @@ { "connectionCheckOutStartedEvent": {} }, + { + "connectionCheckOutStartedEvent": {} + }, { "connectionCheckOutStartedEvent": {} } @@ -1775,6 +1829,9 @@ { "connectionCheckOutStartedEvent": {} }, + { + "connectionCheckOutStartedEvent": {} + }, { "connectionCheckOutStartedEvent": {} } @@ -1862,6 +1919,9 @@ { "connectionCheckOutStartedEvent": {} }, + { + "connectionCheckOutStartedEvent": {} + }, { "connectionCheckOutStartedEvent": {} } @@ -1953,6 +2013,9 @@ { "connectionCheckOutStartedEvent": {} }, + { + "connectionCheckOutStartedEvent": {} + }, { "connectionCheckOutStartedEvent": {} } @@ -2044,6 +2107,9 @@ { "connectionCheckOutStartedEvent": {} }, + { + "connectionCheckOutStartedEvent": {} + }, { "connectionCheckOutStartedEvent": {} } @@ -2134,6 +2200,9 @@ { "connectionCheckOutStartedEvent": {} }, + { + "connectionCheckOutStartedEvent": {} + }, { "connectionCheckOutStartedEvent": {} } @@ -2224,6 +2293,9 @@ { "connectionCheckOutStartedEvent": {} }, + { + "connectionCheckOutStartedEvent": {} + }, { "connectionCheckOutStartedEvent": {} } @@ -2314,6 +2386,9 @@ { "connectionCheckOutStartedEvent": {} }, + { + "connectionCheckOutStartedEvent": {} + }, { "connectionCheckOutStartedEvent": {} } @@ -2404,6 +2479,9 @@ { "connectionCheckOutStartedEvent": {} }, + { + "connectionCheckOutStartedEvent": {} + }, { "connectionCheckOutStartedEvent": {} } @@ -2491,6 +2569,9 @@ { "connectionCheckOutStartedEvent": {} }, + { + "connectionCheckOutStartedEvent": {} + }, { "connectionCheckOutStartedEvent": {} } @@ -2578,6 +2659,9 @@ { "connectionCheckOutStartedEvent": {} }, + { + "connectionCheckOutStartedEvent": {} + }, { "connectionCheckOutStartedEvent": {} } @@ -2665,6 +2749,9 @@ { "connectionCheckOutStartedEvent": {} }, + { + "connectionCheckOutStartedEvent": {} + }, { "connectionCheckOutStartedEvent": {} } @@ -2752,6 +2839,9 @@ { "connectionCheckOutStartedEvent": {} }, + { + "connectionCheckOutStartedEvent": {} + }, { "connectionCheckOutStartedEvent": {} } @@ -2848,6 +2938,9 @@ { "connectionCheckOutStartedEvent": {} }, + { + "connectionCheckOutStartedEvent": {} + }, { "connectionCheckOutStartedEvent": {} } @@ -2944,6 +3037,9 @@ { "connectionCheckOutStartedEvent": {} }, + { + "connectionCheckOutStartedEvent": {} + }, { "connectionCheckOutStartedEvent": {} } diff --git a/test/retryable_writes/unified/handshakeError.json b/test/retryable_writes/unified/handshakeError.json index bcda0a0bfc..93cb2e849e 100644 --- a/test/retryable_writes/unified/handshakeError.json +++ b/test/retryable_writes/unified/handshakeError.json @@ -124,6 +124,9 @@ { "connectionCheckOutStartedEvent": {} }, + { + "connectionCheckOutStartedEvent": {} + }, { "connectionCheckOutStartedEvent": {} } @@ -230,6 +233,9 @@ { "connectionCheckOutStartedEvent": {} }, + { + "connectionCheckOutStartedEvent": {} + }, { "connectionCheckOutStartedEvent": {} } @@ -323,6 +329,9 @@ { "connectionCheckOutStartedEvent": {} }, + { + "connectionCheckOutStartedEvent": {} + }, { "connectionCheckOutStartedEvent": {} } @@ -416,6 +425,9 @@ { "connectionCheckOutStartedEvent": {} }, + { + "connectionCheckOutStartedEvent": {} + }, { "connectionCheckOutStartedEvent": {} } @@ -511,6 +523,9 @@ { "connectionCheckOutStartedEvent": {} }, + { + "connectionCheckOutStartedEvent": {} + }, { "connectionCheckOutStartedEvent": {} } @@ -606,6 +621,9 @@ { "connectionCheckOutStartedEvent": {} }, + { + "connectionCheckOutStartedEvent": {} + }, { "connectionCheckOutStartedEvent": {} } @@ -696,6 +714,9 @@ { "connectionCheckOutStartedEvent": {} }, + { + "connectionCheckOutStartedEvent": {} + }, { "connectionCheckOutStartedEvent": {} } @@ -786,6 +807,9 @@ { "connectionCheckOutStartedEvent": {} }, + { + "connectionCheckOutStartedEvent": {} + }, { "connectionCheckOutStartedEvent": {} } @@ -879,6 +903,9 @@ { "connectionCheckOutStartedEvent": {} }, + { + "connectionCheckOutStartedEvent": {} + }, { "connectionCheckOutStartedEvent": {} } @@ -972,6 +999,9 @@ { "connectionCheckOutStartedEvent": {} }, + { + "connectionCheckOutStartedEvent": {} + }, { "connectionCheckOutStartedEvent": {} } @@ -1067,6 +1097,9 @@ { "connectionCheckOutStartedEvent": {} }, + { + "connectionCheckOutStartedEvent": {} + }, { "connectionCheckOutStartedEvent": {} } @@ -1162,6 +1195,9 @@ { "connectionCheckOutStartedEvent": {} }, + { + "connectionCheckOutStartedEvent": {} + }, { "connectionCheckOutStartedEvent": {} } @@ -1252,6 +1288,9 @@ { "connectionCheckOutStartedEvent": {} }, + { + "connectionCheckOutStartedEvent": {} + }, { "connectionCheckOutStartedEvent": {} } @@ -1342,6 +1381,9 @@ { "connectionCheckOutStartedEvent": {} }, + { + "connectionCheckOutStartedEvent": {} + }, { "connectionCheckOutStartedEvent": {} } @@ -1435,6 +1477,9 @@ { "connectionCheckOutStartedEvent": {} }, + { + "connectionCheckOutStartedEvent": {} + }, { "connectionCheckOutStartedEvent": {} } @@ -1528,6 +1573,9 @@ { "connectionCheckOutStartedEvent": {} }, + { + "connectionCheckOutStartedEvent": {} + }, { "connectionCheckOutStartedEvent": {} } @@ -1623,6 +1671,9 @@ { "connectionCheckOutStartedEvent": {} }, + { + "connectionCheckOutStartedEvent": {} + }, { "connectionCheckOutStartedEvent": {} } @@ -1718,6 +1769,9 @@ { "connectionCheckOutStartedEvent": {} }, + { + "connectionCheckOutStartedEvent": {} + }, { "connectionCheckOutStartedEvent": {} } @@ -1817,6 +1871,9 @@ { "connectionCheckOutStartedEvent": {} }, + { + "connectionCheckOutStartedEvent": {} + }, { "connectionCheckOutStartedEvent": {} } @@ -1916,6 +1973,9 @@ { "connectionCheckOutStartedEvent": {} }, + { + "connectionCheckOutStartedEvent": {} + }, { "connectionCheckOutStartedEvent": {} } From 771570ddcaf3dc40577cfee6b40e60a5558314b1 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 5 Sep 2025 10:31:08 -0500 Subject: [PATCH 40/53] update maxConnecting test --- test/asynchronous/test_pooling.py | 4 +- test/test_pooling.py | 4 +- uv.lock | 98 ++++++++++++++++++------------- 3 files changed, 61 insertions(+), 45 deletions(-) diff --git a/test/asynchronous/test_pooling.py b/test/asynchronous/test_pooling.py index 1d0028f43c..4949faafbc 100644 --- a/test/asynchronous/test_pooling.py +++ b/test/asynchronous/test_pooling.py @@ -586,11 +586,11 @@ async def test_pool_backoff_preserves_existing_connections(self): await pool.close() async def test_pool_backoff_limits_maxConnecting(self): - client = await self.async_rs_or_single_client() + client = await self.async_rs_or_single_client(maxConnecting=30) _, baseline_conns = await self._check_maxConnecting(client) await client.close() - client = await self.async_rs_or_single_client() + client = await self.async_rs_or_single_client(maxConnecting=30) _, backoff_conns = await self._check_maxConnecting(client, backoff=True) await client.close() diff --git a/test/test_pooling.py b/test/test_pooling.py index f0d3140c66..26315354cd 100644 --- a/test/test_pooling.py +++ b/test/test_pooling.py @@ -582,11 +582,11 @@ def test_pool_backoff_preserves_existing_connections(self): pool.close() def test_pool_backoff_limits_maxConnecting(self): - client = self.rs_or_single_client() + client = self.rs_or_single_client(maxConnecting=30) _, baseline_conns = self._check_maxConnecting(client) client.close() - client = self.rs_or_single_client() + client = self.rs_or_single_client(maxConnecting=30) _, backoff_conns = self._check_maxConnecting(client, backoff=True) client.close() diff --git a/uv.lock b/uv.lock index 9c45c4cdb9..9c74220460 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 2 +revision = 3 requires-python = ">=3.9" resolution-markers = [ "python_full_version == '3.14.*'", @@ -1047,46 +1047,53 @@ dependencies = [ [[package]] name = "mypy" -version = "1.14.1" +version = "1.17.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "mypy-extensions" }, + { name = "pathspec" }, { name = "tomli", marker = "python_full_version < '3.11'" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/b9/eb/2c92d8ea1e684440f54fa49ac5d9a5f19967b7b472a281f419e69a8d228e/mypy-1.14.1.tar.gz", hash = "sha256:7ec88144fe9b510e8475ec2f5f251992690fcf89ccb4500b214b4226abcd32d6", size = 3216051, upload-time = "2024-12-30T16:39:07.335Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9b/7a/87ae2adb31d68402da6da1e5f30c07ea6063e9f09b5e7cfc9dfa44075e74/mypy-1.14.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:52686e37cf13d559f668aa398dd7ddf1f92c5d613e4f8cb262be2fb4fedb0fcb", size = 11211002, upload-time = "2024-12-30T16:37:22.435Z" }, - { url = "https://files.pythonhosted.org/packages/e1/23/eada4c38608b444618a132be0d199b280049ded278b24cbb9d3fc59658e4/mypy-1.14.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:1fb545ca340537d4b45d3eecdb3def05e913299ca72c290326be19b3804b39c0", size = 10358400, upload-time = "2024-12-30T16:37:53.526Z" }, - { url = "https://files.pythonhosted.org/packages/43/c9/d6785c6f66241c62fd2992b05057f404237deaad1566545e9f144ced07f5/mypy-1.14.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:90716d8b2d1f4cd503309788e51366f07c56635a3309b0f6a32547eaaa36a64d", size = 12095172, upload-time = "2024-12-30T16:37:50.332Z" }, - { url = "https://files.pythonhosted.org/packages/c3/62/daa7e787770c83c52ce2aaf1a111eae5893de9e004743f51bfcad9e487ec/mypy-1.14.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2ae753f5c9fef278bcf12e1a564351764f2a6da579d4a81347e1d5a15819997b", size = 12828732, upload-time = "2024-12-30T16:37:29.96Z" }, - { url = "https://files.pythonhosted.org/packages/1b/a2/5fb18318a3637f29f16f4e41340b795da14f4751ef4f51c99ff39ab62e52/mypy-1.14.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:e0fe0f5feaafcb04505bcf439e991c6d8f1bf8b15f12b05feeed96e9e7bf1427", size = 13012197, upload-time = "2024-12-30T16:38:05.037Z" }, - { url = "https://files.pythonhosted.org/packages/28/99/e153ce39105d164b5f02c06c35c7ba958aaff50a2babba7d080988b03fe7/mypy-1.14.1-cp310-cp310-win_amd64.whl", hash = "sha256:7d54bd85b925e501c555a3227f3ec0cfc54ee8b6930bd6141ec872d1c572f81f", size = 9780836, upload-time = "2024-12-30T16:37:19.726Z" }, - { url = "https://files.pythonhosted.org/packages/da/11/a9422850fd506edbcdc7f6090682ecceaf1f87b9dd847f9df79942da8506/mypy-1.14.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f995e511de847791c3b11ed90084a7a0aafdc074ab88c5a9711622fe4751138c", size = 11120432, upload-time = "2024-12-30T16:37:11.533Z" }, - { url = "https://files.pythonhosted.org/packages/b6/9e/47e450fd39078d9c02d620545b2cb37993a8a8bdf7db3652ace2f80521ca/mypy-1.14.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d64169ec3b8461311f8ce2fd2eb5d33e2d0f2c7b49116259c51d0d96edee48d1", size = 10279515, upload-time = "2024-12-30T16:37:40.724Z" }, - { url = "https://files.pythonhosted.org/packages/01/b5/6c8d33bd0f851a7692a8bfe4ee75eb82b6983a3cf39e5e32a5d2a723f0c1/mypy-1.14.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ba24549de7b89b6381b91fbc068d798192b1b5201987070319889e93038967a8", size = 12025791, upload-time = "2024-12-30T16:36:58.73Z" }, - { url = "https://files.pythonhosted.org/packages/f0/4c/e10e2c46ea37cab5c471d0ddaaa9a434dc1d28650078ac1b56c2d7b9b2e4/mypy-1.14.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:183cf0a45457d28ff9d758730cd0210419ac27d4d3f285beda038c9083363b1f", size = 12749203, upload-time = "2024-12-30T16:37:03.741Z" }, - { url = "https://files.pythonhosted.org/packages/88/55/beacb0c69beab2153a0f57671ec07861d27d735a0faff135a494cd4f5020/mypy-1.14.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f2a0ecc86378f45347f586e4163d1769dd81c5a223d577fe351f26b179e148b1", size = 12885900, upload-time = "2024-12-30T16:37:57.948Z" }, - { url = "https://files.pythonhosted.org/packages/a2/75/8c93ff7f315c4d086a2dfcde02f713004357d70a163eddb6c56a6a5eff40/mypy-1.14.1-cp311-cp311-win_amd64.whl", hash = "sha256:ad3301ebebec9e8ee7135d8e3109ca76c23752bac1e717bc84cd3836b4bf3eae", size = 9777869, upload-time = "2024-12-30T16:37:33.428Z" }, - { url = "https://files.pythonhosted.org/packages/43/1b/b38c079609bb4627905b74fc6a49849835acf68547ac33d8ceb707de5f52/mypy-1.14.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:30ff5ef8519bbc2e18b3b54521ec319513a26f1bba19a7582e7b1f58a6e69f14", size = 11266668, upload-time = "2024-12-30T16:38:02.211Z" }, - { url = "https://files.pythonhosted.org/packages/6b/75/2ed0d2964c1ffc9971c729f7a544e9cd34b2cdabbe2d11afd148d7838aa2/mypy-1.14.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:cb9f255c18052343c70234907e2e532bc7e55a62565d64536dbc7706a20b78b9", size = 10254060, upload-time = "2024-12-30T16:37:46.131Z" }, - { url = "https://files.pythonhosted.org/packages/a1/5f/7b8051552d4da3c51bbe8fcafffd76a6823779101a2b198d80886cd8f08e/mypy-1.14.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8b4e3413e0bddea671012b063e27591b953d653209e7a4fa5e48759cda77ca11", size = 11933167, upload-time = "2024-12-30T16:37:43.534Z" }, - { url = "https://files.pythonhosted.org/packages/04/90/f53971d3ac39d8b68bbaab9a4c6c58c8caa4d5fd3d587d16f5927eeeabe1/mypy-1.14.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:553c293b1fbdebb6c3c4030589dab9fafb6dfa768995a453d8a5d3b23784af2e", size = 12864341, upload-time = "2024-12-30T16:37:36.249Z" }, - { url = "https://files.pythonhosted.org/packages/03/d2/8bc0aeaaf2e88c977db41583559319f1821c069e943ada2701e86d0430b7/mypy-1.14.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fad79bfe3b65fe6a1efaed97b445c3d37f7be9fdc348bdb2d7cac75579607c89", size = 12972991, upload-time = "2024-12-30T16:37:06.743Z" }, - { url = "https://files.pythonhosted.org/packages/6f/17/07815114b903b49b0f2cf7499f1c130e5aa459411596668267535fe9243c/mypy-1.14.1-cp312-cp312-win_amd64.whl", hash = "sha256:8fa2220e54d2946e94ab6dbb3ba0a992795bd68b16dc852db33028df2b00191b", size = 9879016, upload-time = "2024-12-30T16:37:15.02Z" }, - { url = "https://files.pythonhosted.org/packages/9e/15/bb6a686901f59222275ab228453de741185f9d54fecbaacec041679496c6/mypy-1.14.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:92c3ed5afb06c3a8e188cb5da4984cab9ec9a77ba956ee419c68a388b4595255", size = 11252097, upload-time = "2024-12-30T16:37:25.144Z" }, - { url = "https://files.pythonhosted.org/packages/f8/b3/8b0f74dfd072c802b7fa368829defdf3ee1566ba74c32a2cb2403f68024c/mypy-1.14.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:dbec574648b3e25f43d23577309b16534431db4ddc09fda50841f1e34e64ed34", size = 10239728, upload-time = "2024-12-30T16:38:08.634Z" }, - { url = "https://files.pythonhosted.org/packages/c5/9b/4fd95ab20c52bb5b8c03cc49169be5905d931de17edfe4d9d2986800b52e/mypy-1.14.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8c6d94b16d62eb3e947281aa7347d78236688e21081f11de976376cf010eb31a", size = 11924965, upload-time = "2024-12-30T16:38:12.132Z" }, - { url = "https://files.pythonhosted.org/packages/56/9d/4a236b9c57f5d8f08ed346914b3f091a62dd7e19336b2b2a0d85485f82ff/mypy-1.14.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d4b19b03fdf54f3c5b2fa474c56b4c13c9dbfb9a2db4370ede7ec11a2c5927d9", size = 12867660, upload-time = "2024-12-30T16:38:17.342Z" }, - { url = "https://files.pythonhosted.org/packages/40/88/a61a5497e2f68d9027de2bb139c7bb9abaeb1be1584649fa9d807f80a338/mypy-1.14.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:0c911fde686394753fff899c409fd4e16e9b294c24bfd5e1ea4675deae1ac6fd", size = 12969198, upload-time = "2024-12-30T16:38:32.839Z" }, - { url = "https://files.pythonhosted.org/packages/54/da/3d6fc5d92d324701b0c23fb413c853892bfe0e1dbe06c9138037d459756b/mypy-1.14.1-cp313-cp313-win_amd64.whl", hash = "sha256:8b21525cb51671219f5307be85f7e646a153e5acc656e5cebf64bfa076c50107", size = 9885276, upload-time = "2024-12-30T16:38:20.828Z" }, - { url = "https://files.pythonhosted.org/packages/ca/1f/186d133ae2514633f8558e78cd658070ba686c0e9275c5a5c24a1e1f0d67/mypy-1.14.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:3888a1816d69f7ab92092f785a462944b3ca16d7c470d564165fe703b0970c35", size = 11200493, upload-time = "2024-12-30T16:38:26.935Z" }, - { url = "https://files.pythonhosted.org/packages/af/fc/4842485d034e38a4646cccd1369f6b1ccd7bc86989c52770d75d719a9941/mypy-1.14.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:46c756a444117c43ee984bd055db99e498bc613a70bbbc120272bd13ca579fbc", size = 10357702, upload-time = "2024-12-30T16:38:50.623Z" }, - { url = "https://files.pythonhosted.org/packages/b4/e6/457b83f2d701e23869cfec013a48a12638f75b9d37612a9ddf99072c1051/mypy-1.14.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:27fc248022907e72abfd8e22ab1f10e903915ff69961174784a3900a8cba9ad9", size = 12091104, upload-time = "2024-12-30T16:38:53.735Z" }, - { url = "https://files.pythonhosted.org/packages/f1/bf/76a569158db678fee59f4fd30b8e7a0d75bcbaeef49edd882a0d63af6d66/mypy-1.14.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:499d6a72fb7e5de92218db961f1a66d5f11783f9ae549d214617edab5d4dbdbb", size = 12830167, upload-time = "2024-12-30T16:38:56.437Z" }, - { url = "https://files.pythonhosted.org/packages/43/bc/0bc6b694b3103de9fed61867f1c8bd33336b913d16831431e7cb48ef1c92/mypy-1.14.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:57961db9795eb566dc1d1b4e9139ebc4c6b0cb6e7254ecde69d1552bf7613f60", size = 13013834, upload-time = "2024-12-30T16:38:59.204Z" }, - { url = "https://files.pythonhosted.org/packages/b0/79/5f5ec47849b6df1e6943d5fd8e6632fbfc04b4fd4acfa5a5a9535d11b4e2/mypy-1.14.1-cp39-cp39-win_amd64.whl", hash = "sha256:07ba89fdcc9451f2ebb02853deb6aaaa3d2239a236669a63ab3801bbf923ef5c", size = 9781231, upload-time = "2024-12-30T16:39:05.124Z" }, - { url = "https://files.pythonhosted.org/packages/a0/b5/32dd67b69a16d088e533962e5044e51004176a9952419de0370cdaead0f8/mypy-1.14.1-py3-none-any.whl", hash = "sha256:b66a60cc4073aeb8ae00057f9c1f64d49e90f918fbcef9a977eb121da8b8f1d1", size = 2752905, upload-time = "2024-12-30T16:38:42.021Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/8e/22/ea637422dedf0bf36f3ef238eab4e455e2a0dcc3082b5cc067615347ab8e/mypy-1.17.1.tar.gz", hash = "sha256:25e01ec741ab5bb3eec8ba9cdb0f769230368a22c959c4937360efb89b7e9f01", size = 3352570, upload-time = "2025-07-31T07:54:19.204Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/77/a9/3d7aa83955617cdf02f94e50aab5c830d205cfa4320cf124ff64acce3a8e/mypy-1.17.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3fbe6d5555bf608c47203baa3e72dbc6ec9965b3d7c318aa9a4ca76f465bd972", size = 11003299, upload-time = "2025-07-31T07:54:06.425Z" }, + { url = "https://files.pythonhosted.org/packages/83/e8/72e62ff837dd5caaac2b4a5c07ce769c8e808a00a65e5d8f94ea9c6f20ab/mypy-1.17.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:80ef5c058b7bce08c83cac668158cb7edea692e458d21098c7d3bce35a5d43e7", size = 10125451, upload-time = "2025-07-31T07:53:52.974Z" }, + { url = "https://files.pythonhosted.org/packages/7d/10/f3f3543f6448db11881776f26a0ed079865926b0c841818ee22de2c6bbab/mypy-1.17.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c4a580f8a70c69e4a75587bd925d298434057fe2a428faaf927ffe6e4b9a98df", size = 11916211, upload-time = "2025-07-31T07:53:18.879Z" }, + { url = "https://files.pythonhosted.org/packages/06/bf/63e83ed551282d67bb3f7fea2cd5561b08d2bb6eb287c096539feb5ddbc5/mypy-1.17.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dd86bb649299f09d987a2eebb4d52d10603224500792e1bee18303bbcc1ce390", size = 12652687, upload-time = "2025-07-31T07:53:30.544Z" }, + { url = "https://files.pythonhosted.org/packages/69/66/68f2eeef11facf597143e85b694a161868b3b006a5fbad50e09ea117ef24/mypy-1.17.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:a76906f26bd8d51ea9504966a9c25419f2e668f012e0bdf3da4ea1526c534d94", size = 12896322, upload-time = "2025-07-31T07:53:50.74Z" }, + { url = "https://files.pythonhosted.org/packages/a3/87/8e3e9c2c8bd0d7e071a89c71be28ad088aaecbadf0454f46a540bda7bca6/mypy-1.17.1-cp310-cp310-win_amd64.whl", hash = "sha256:e79311f2d904ccb59787477b7bd5d26f3347789c06fcd7656fa500875290264b", size = 9507962, upload-time = "2025-07-31T07:53:08.431Z" }, + { url = "https://files.pythonhosted.org/packages/46/cf/eadc80c4e0a70db1c08921dcc220357ba8ab2faecb4392e3cebeb10edbfa/mypy-1.17.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ad37544be07c5d7fba814eb370e006df58fed8ad1ef33ed1649cb1889ba6ff58", size = 10921009, upload-time = "2025-07-31T07:53:23.037Z" }, + { url = "https://files.pythonhosted.org/packages/5d/c1/c869d8c067829ad30d9bdae051046561552516cfb3a14f7f0347b7d973ee/mypy-1.17.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:064e2ff508e5464b4bd807a7c1625bc5047c5022b85c70f030680e18f37273a5", size = 10047482, upload-time = "2025-07-31T07:53:26.151Z" }, + { url = "https://files.pythonhosted.org/packages/98/b9/803672bab3fe03cee2e14786ca056efda4bb511ea02dadcedde6176d06d0/mypy-1.17.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:70401bbabd2fa1aa7c43bb358f54037baf0586f41e83b0ae67dd0534fc64edfd", size = 11832883, upload-time = "2025-07-31T07:53:47.948Z" }, + { url = "https://files.pythonhosted.org/packages/88/fb/fcdac695beca66800918c18697b48833a9a6701de288452b6715a98cfee1/mypy-1.17.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e92bdc656b7757c438660f775f872a669b8ff374edc4d18277d86b63edba6b8b", size = 12566215, upload-time = "2025-07-31T07:54:04.031Z" }, + { url = "https://files.pythonhosted.org/packages/7f/37/a932da3d3dace99ee8eb2043b6ab03b6768c36eb29a02f98f46c18c0da0e/mypy-1.17.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c1fdf4abb29ed1cb091cf432979e162c208a5ac676ce35010373ff29247bcad5", size = 12751956, upload-time = "2025-07-31T07:53:36.263Z" }, + { url = "https://files.pythonhosted.org/packages/8c/cf/6438a429e0f2f5cab8bc83e53dbebfa666476f40ee322e13cac5e64b79e7/mypy-1.17.1-cp311-cp311-win_amd64.whl", hash = "sha256:ff2933428516ab63f961644bc49bc4cbe42bbffb2cd3b71cc7277c07d16b1a8b", size = 9507307, upload-time = "2025-07-31T07:53:59.734Z" }, + { url = "https://files.pythonhosted.org/packages/17/a2/7034d0d61af8098ec47902108553122baa0f438df8a713be860f7407c9e6/mypy-1.17.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:69e83ea6553a3ba79c08c6e15dbd9bfa912ec1e493bf75489ef93beb65209aeb", size = 11086295, upload-time = "2025-07-31T07:53:28.124Z" }, + { url = "https://files.pythonhosted.org/packages/14/1f/19e7e44b594d4b12f6ba8064dbe136505cec813549ca3e5191e40b1d3cc2/mypy-1.17.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1b16708a66d38abb1e6b5702f5c2c87e133289da36f6a1d15f6a5221085c6403", size = 10112355, upload-time = "2025-07-31T07:53:21.121Z" }, + { url = "https://files.pythonhosted.org/packages/5b/69/baa33927e29e6b4c55d798a9d44db5d394072eef2bdc18c3e2048c9ed1e9/mypy-1.17.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:89e972c0035e9e05823907ad5398c5a73b9f47a002b22359b177d40bdaee7056", size = 11875285, upload-time = "2025-07-31T07:53:55.293Z" }, + { url = "https://files.pythonhosted.org/packages/90/13/f3a89c76b0a41e19490b01e7069713a30949d9a6c147289ee1521bcea245/mypy-1.17.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:03b6d0ed2b188e35ee6d5c36b5580cffd6da23319991c49ab5556c023ccf1341", size = 12737895, upload-time = "2025-07-31T07:53:43.623Z" }, + { url = "https://files.pythonhosted.org/packages/23/a1/c4ee79ac484241301564072e6476c5a5be2590bc2e7bfd28220033d2ef8f/mypy-1.17.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c837b896b37cd103570d776bda106eabb8737aa6dd4f248451aecf53030cdbeb", size = 12931025, upload-time = "2025-07-31T07:54:17.125Z" }, + { url = "https://files.pythonhosted.org/packages/89/b8/7409477be7919a0608900e6320b155c72caab4fef46427c5cc75f85edadd/mypy-1.17.1-cp312-cp312-win_amd64.whl", hash = "sha256:665afab0963a4b39dff7c1fa563cc8b11ecff7910206db4b2e64dd1ba25aed19", size = 9584664, upload-time = "2025-07-31T07:54:12.842Z" }, + { url = "https://files.pythonhosted.org/packages/5b/82/aec2fc9b9b149f372850291827537a508d6c4d3664b1750a324b91f71355/mypy-1.17.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:93378d3203a5c0800c6b6d850ad2f19f7a3cdf1a3701d3416dbf128805c6a6a7", size = 11075338, upload-time = "2025-07-31T07:53:38.873Z" }, + { url = "https://files.pythonhosted.org/packages/07/ac/ee93fbde9d2242657128af8c86f5d917cd2887584cf948a8e3663d0cd737/mypy-1.17.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:15d54056f7fe7a826d897789f53dd6377ec2ea8ba6f776dc83c2902b899fee81", size = 10113066, upload-time = "2025-07-31T07:54:14.707Z" }, + { url = "https://files.pythonhosted.org/packages/5a/68/946a1e0be93f17f7caa56c45844ec691ca153ee8b62f21eddda336a2d203/mypy-1.17.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:209a58fed9987eccc20f2ca94afe7257a8f46eb5df1fb69958650973230f91e6", size = 11875473, upload-time = "2025-07-31T07:53:14.504Z" }, + { url = "https://files.pythonhosted.org/packages/9f/0f/478b4dce1cb4f43cf0f0d00fba3030b21ca04a01b74d1cd272a528cf446f/mypy-1.17.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:099b9a5da47de9e2cb5165e581f158e854d9e19d2e96b6698c0d64de911dd849", size = 12744296, upload-time = "2025-07-31T07:53:03.896Z" }, + { url = "https://files.pythonhosted.org/packages/ca/70/afa5850176379d1b303f992a828de95fc14487429a7139a4e0bdd17a8279/mypy-1.17.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:fa6ffadfbe6994d724c5a1bb6123a7d27dd68fc9c059561cd33b664a79578e14", size = 12914657, upload-time = "2025-07-31T07:54:08.576Z" }, + { url = "https://files.pythonhosted.org/packages/53/f9/4a83e1c856a3d9c8f6edaa4749a4864ee98486e9b9dbfbc93842891029c2/mypy-1.17.1-cp313-cp313-win_amd64.whl", hash = "sha256:9a2b7d9180aed171f033c9f2fc6c204c1245cf60b0cb61cf2e7acc24eea78e0a", size = 9593320, upload-time = "2025-07-31T07:53:01.341Z" }, + { url = "https://files.pythonhosted.org/packages/38/56/79c2fac86da57c7d8c48622a05873eaab40b905096c33597462713f5af90/mypy-1.17.1-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:15a83369400454c41ed3a118e0cc58bd8123921a602f385cb6d6ea5df050c733", size = 11040037, upload-time = "2025-07-31T07:54:10.942Z" }, + { url = "https://files.pythonhosted.org/packages/4d/c3/adabe6ff53638e3cad19e3547268482408323b1e68bf082c9119000cd049/mypy-1.17.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:55b918670f692fc9fba55c3298d8a3beae295c5cded0a55dccdc5bbead814acd", size = 10131550, upload-time = "2025-07-31T07:53:41.307Z" }, + { url = "https://files.pythonhosted.org/packages/b8/c5/2e234c22c3bdeb23a7817af57a58865a39753bde52c74e2c661ee0cfc640/mypy-1.17.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:62761474061feef6f720149d7ba876122007ddc64adff5ba6f374fda35a018a0", size = 11872963, upload-time = "2025-07-31T07:53:16.878Z" }, + { url = "https://files.pythonhosted.org/packages/ab/26/c13c130f35ca8caa5f2ceab68a247775648fdcd6c9a18f158825f2bc2410/mypy-1.17.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c49562d3d908fd49ed0938e5423daed8d407774a479b595b143a3d7f87cdae6a", size = 12710189, upload-time = "2025-07-31T07:54:01.962Z" }, + { url = "https://files.pythonhosted.org/packages/82/df/c7d79d09f6de8383fe800521d066d877e54d30b4fb94281c262be2df84ef/mypy-1.17.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:397fba5d7616a5bc60b45c7ed204717eaddc38f826e3645402c426057ead9a91", size = 12900322, upload-time = "2025-07-31T07:53:10.551Z" }, + { url = "https://files.pythonhosted.org/packages/b8/98/3d5a48978b4f708c55ae832619addc66d677f6dc59f3ebad71bae8285ca6/mypy-1.17.1-cp314-cp314-win_amd64.whl", hash = "sha256:9d6b20b97d373f41617bd0708fd46aa656059af57f2ef72aa8c7d6a2b73b74ed", size = 9751879, upload-time = "2025-07-31T07:52:56.683Z" }, + { url = "https://files.pythonhosted.org/packages/29/cb/673e3d34e5d8de60b3a61f44f80150a738bff568cd6b7efb55742a605e98/mypy-1.17.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5d1092694f166a7e56c805caaf794e0585cabdbf1df36911c414e4e9abb62ae9", size = 10992466, upload-time = "2025-07-31T07:53:57.574Z" }, + { url = "https://files.pythonhosted.org/packages/0c/d0/fe1895836eea3a33ab801561987a10569df92f2d3d4715abf2cfeaa29cb2/mypy-1.17.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:79d44f9bfb004941ebb0abe8eff6504223a9c1ac51ef967d1263c6572bbebc99", size = 10117638, upload-time = "2025-07-31T07:53:34.256Z" }, + { url = "https://files.pythonhosted.org/packages/97/f3/514aa5532303aafb95b9ca400a31054a2bd9489de166558c2baaeea9c522/mypy-1.17.1-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b01586eed696ec905e61bd2568f48740f7ac4a45b3a468e6423a03d3788a51a8", size = 11915673, upload-time = "2025-07-31T07:52:59.361Z" }, + { url = "https://files.pythonhosted.org/packages/ab/c3/c0805f0edec96fe8e2c048b03769a6291523d509be8ee7f56ae922fa3882/mypy-1.17.1-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:43808d9476c36b927fbcd0b0255ce75efe1b68a080154a38ae68a7e62de8f0f8", size = 12649022, upload-time = "2025-07-31T07:53:45.92Z" }, + { url = "https://files.pythonhosted.org/packages/45/3e/d646b5a298ada21a8512fa7e5531f664535a495efa672601702398cea2b4/mypy-1.17.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:feb8cc32d319edd5859da2cc084493b3e2ce5e49a946377663cc90f6c15fb259", size = 12895536, upload-time = "2025-07-31T07:53:06.17Z" }, + { url = "https://files.pythonhosted.org/packages/14/55/e13d0dcd276975927d1f4e9e2ec4fd409e199f01bdc671717e673cc63a22/mypy-1.17.1-cp39-cp39-win_amd64.whl", hash = "sha256:d7598cf74c3e16539d4e2f0b8d8c318e00041553d83d4861f87c7a72e95ac24d", size = 9512564, upload-time = "2025-07-31T07:53:12.346Z" }, + { url = "https://files.pythonhosted.org/packages/1d/f3/8fcd2af0f5b806f6cf463efaffd3c9548a28f84220493ecd38d127b6b66d/mypy-1.17.1-py3-none-any.whl", hash = "sha256:a9f52c0351c21fe24c21d8c0eb1f62967b262d6729393397b6f443c3b773c3b9", size = 2283411, upload-time = "2025-07-31T07:53:24.664Z" }, ] [[package]] @@ -1116,6 +1123,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484", size = 66469, upload-time = "2025-04-19T11:48:57.875Z" }, ] +[[package]] +name = "pathspec" +version = "0.12.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ca/bc/f35b8446f4531a7cb215605d100cd88b7ac6f44ab3fc94870c120ab3adbf/pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712", size = 51043, upload-time = "2023-12-10T22:30:45Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cc/20/ff623b09d963f88bfde16306a54e12ee5ea43e9b597108672ff3a408aad6/pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08", size = 31191, upload-time = "2023-12-10T22:30:43.14Z" }, +] + [[package]] name = "pip" version = "25.2" @@ -1315,7 +1331,7 @@ provides-extras = ["aws", "docs", "encryption", "gssapi", "ocsp", "snappy", "tes [package.metadata.requires-dev] coverage = [ - { name = "coverage", specifier = ">=5,<=7.5" }, + { name = "coverage", specifier = ">=5,<=7.10.3" }, { name = "pytest-cov" }, ] dev = [{ name = "pre-commit", specifier = ">=4.0" }] @@ -1329,9 +1345,9 @@ perf = [{ name = "simplejson" }] pip = [{ name = "pip" }] pymongocrypt-source = [{ name = "pymongocrypt", git = "https://github.com/mongodb/libmongocrypt?subdirectory=bindings%2Fpython&rev=master" }] typing = [ - { name = "mypy", specifier = "==1.14.1" }, + { name = "mypy", specifier = "==1.17.1" }, { name = "pip" }, - { name = "pyright", specifier = "==1.1.392.post0" }, + { name = "pyright", specifier = "==1.1.403" }, { name = "typing-extensions" }, ] @@ -1375,15 +1391,15 @@ wheels = [ [[package]] name = "pyright" -version = "1.1.392.post0" +version = "1.1.403" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "nodeenv" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/66/df/3c6f6b08fba7ccf49b114dfc4bb33e25c299883fd763f93fad47ef8bc58d/pyright-1.1.392.post0.tar.gz", hash = "sha256:3b7f88de74a28dcfa90c7d90c782b6569a48c2be5f9d4add38472bdaac247ebd", size = 3789911, upload-time = "2025-01-15T15:01:20.913Z" } +sdist = { url = "https://files.pythonhosted.org/packages/fe/f6/35f885264ff08c960b23d1542038d8da86971c5d8c955cfab195a4f672d7/pyright-1.1.403.tar.gz", hash = "sha256:3ab69b9f41c67fb5bbb4d7a36243256f0d549ed3608678d381d5f51863921104", size = 3913526, upload-time = "2025-07-09T07:15:52.882Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/e7/b1/a18de17f40e4f61ca58856b9ef9b0febf74ff88978c3f7776f910071f567/pyright-1.1.392.post0-py3-none-any.whl", hash = "sha256:252f84458a46fa2f0fd4e2f91fc74f50b9ca52c757062e93f6c250c0d8329eb2", size = 5595487, upload-time = "2025-01-15T15:01:17.775Z" }, + { url = "https://files.pythonhosted.org/packages/49/b6/b04e5c2f41a5ccad74a1a4759da41adb20b4bc9d59a5e08d29ba60084d07/pyright-1.1.403-py3-none-any.whl", hash = "sha256:c0eeca5aa76cbef3fcc271259bbd785753c7ad7bcac99a9162b4c4c7daed23b3", size = 5684504, upload-time = "2025-07-09T07:15:50.958Z" }, ] [[package]] From 6623261d5237296d24d08d4aa5537c66603f5c99 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 5 Sep 2025 12:57:36 -0500 Subject: [PATCH 41/53] fix maxconnecting test --- test/asynchronous/test_pooling.py | 8 +++++--- test/test_pooling.py | 8 +++++--- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/test/asynchronous/test_pooling.py b/test/asynchronous/test_pooling.py index 4949faafbc..9009577f1c 100644 --- a/test/asynchronous/test_pooling.py +++ b/test/asynchronous/test_pooling.py @@ -29,6 +29,7 @@ from pymongo.errors import AutoReconnect, ConnectionFailure, DuplicateKeyError from pymongo.hello import HelloCompat from pymongo.lock import _async_create_lock +from pymongo.read_preferences import ReadPreference sys.path[0:0] = [""] @@ -397,7 +398,8 @@ async def test_checkout_more_than_max_pool_size(self): async def _check_maxConnecting( self, client: AsyncMongoClient, backoff=False ) -> tuple[int, int]: - await client.test.test.insert_one({}) + coll = client.test.test.with_options(read_preference=ReadPreference.PRIMARY) + await coll.insert_one({}) pool = await async_get_pool(client) if backoff: @@ -406,7 +408,7 @@ async def _check_maxConnecting( # Run 50 short running operations async def find_one(): - docs.append(await client.test.test.find_one({})) + docs.append(await coll.find_one({})) tasks = [ConcurrentRunner(target=find_one) for _ in range(50)] for task in tasks: @@ -414,7 +416,7 @@ async def find_one(): for task in tasks: await task.join(10) - await client.test.test.delete_many({}) + await coll.delete_many({}) return len(docs), len(pool.conns) diff --git a/test/test_pooling.py b/test/test_pooling.py index 26315354cd..c9dc6dffc7 100644 --- a/test/test_pooling.py +++ b/test/test_pooling.py @@ -29,6 +29,7 @@ from pymongo.errors import AutoReconnect, ConnectionFailure, DuplicateKeyError from pymongo.hello import HelloCompat from pymongo.lock import _create_lock +from pymongo.read_preferences import ReadPreference sys.path[0:0] = [""] @@ -395,7 +396,8 @@ def test_checkout_more_than_max_pool_size(self): pool.close() def _check_maxConnecting(self, client: MongoClient, backoff=False) -> tuple[int, int]: - client.test.test.insert_one({}) + coll = client.test.test.with_options(read_preference=ReadPreference.PRIMARY) + coll.insert_one({}) pool = get_pool(client) if backoff: @@ -404,7 +406,7 @@ def _check_maxConnecting(self, client: MongoClient, backoff=False) -> tuple[int, # Run 50 short running operations def find_one(): - docs.append(client.test.test.find_one({})) + docs.append(coll.find_one({})) tasks = [ConcurrentRunner(target=find_one) for _ in range(50)] for task in tasks: @@ -412,7 +414,7 @@ def find_one(): for task in tasks: task.join(10) - client.test.test.delete_many({}) + coll.delete_many({}) return len(docs), len(pool.conns) From f602d4c301bae7f943d5ec70462402417e8ea8e0 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 5 Sep 2025 15:59:22 -0500 Subject: [PATCH 42/53] fix handling of maxConnecting --- pymongo/asynchronous/pool.py | 14 ++++++---- pymongo/synchronous/pool.py | 14 ++++++---- test/asynchronous/test_pooling.py | 46 ++++++++++--------------------- test/test_pooling.py | 44 ++++++++++------------------- 4 files changed, 46 insertions(+), 72 deletions(-) diff --git a/pymongo/asynchronous/pool.py b/pymongo/asynchronous/pool.py index ec67899486..6184cb98b7 100644 --- a/pymongo/asynchronous/pool.py +++ b/pymongo/asynchronous/pool.py @@ -788,7 +788,6 @@ def __init__( # Enforces: maxConnecting # Also used for: clearing the wait queue self._max_connecting_cond = _async_create_condition(self.lock) - self._max_connecting = self.opts.max_connecting self._pending = 0 self._client_id = client_id self._backoff = 0 @@ -931,6 +930,11 @@ async def _reset( for conn in sockets: await conn.close_conn(ConnectionClosedReason.STALE) + @property + def max_connecting(self) -> int: + """The current max connecting limit for the pool.""" + return 1 if self._backoff else self.opts.max_connecting + async def update_is_writable(self, is_writable: Optional[bool]) -> None: """Updates the is_writable attribute on all sockets currently in the Pool. @@ -997,8 +1001,7 @@ async def remove_stale_sockets(self, reference_generation: int) -> None: async with self._max_connecting_cond: # If maxConnecting connections are already being created # by this pool then try again later instead of waiting. - max_connecting = 1 if self._backoff else self._max_connecting - if self._pending >= max_connecting: + if self._pending >= self.max_connecting: return self._pending += 1 incremented = True @@ -1297,13 +1300,12 @@ async def _get_conn( # to be checked back into the pool. async with self._max_connecting_cond: self._raise_if_not_ready(checkout_started_time, emit_event=False) - max_connecting = 1 if self._backoff else self._max_connecting - while not (self.conns or self._pending < max_connecting): + while not (self.conns or self._pending < self.max_connecting): timeout = deadline - time.monotonic() if deadline else None if not await _async_cond_wait(self._max_connecting_cond, timeout): # Timed out, notify the next thread to ensure a # timeout doesn't consume the condition. - if self.conns or self._pending < max_connecting: + if self.conns or self._pending < self.max_connecting: self._max_connecting_cond.notify() emitted_event = True self._raise_wait_queue_timeout(checkout_started_time) diff --git a/pymongo/synchronous/pool.py b/pymongo/synchronous/pool.py index fdd12bb728..16476f5c85 100644 --- a/pymongo/synchronous/pool.py +++ b/pymongo/synchronous/pool.py @@ -786,7 +786,6 @@ def __init__( # Enforces: maxConnecting # Also used for: clearing the wait queue self._max_connecting_cond = _create_condition(self.lock) - self._max_connecting = self.opts.max_connecting self._pending = 0 self._client_id = client_id self._backoff = 0 @@ -929,6 +928,11 @@ def _reset( for conn in sockets: conn.close_conn(ConnectionClosedReason.STALE) + @property + def max_connecting(self) -> int: + """The current max connecting limit for the pool.""" + return 1 if self._backoff else self.opts.max_connecting + def update_is_writable(self, is_writable: Optional[bool]) -> None: """Updates the is_writable attribute on all sockets currently in the Pool. @@ -993,8 +997,7 @@ def remove_stale_sockets(self, reference_generation: int) -> None: with self._max_connecting_cond: # If maxConnecting connections are already being created # by this pool then try again later instead of waiting. - max_connecting = 1 if self._backoff else self._max_connecting - if self._pending >= max_connecting: + if self._pending >= self.max_connecting: return self._pending += 1 incremented = True @@ -1293,13 +1296,12 @@ def _get_conn( # to be checked back into the pool. with self._max_connecting_cond: self._raise_if_not_ready(checkout_started_time, emit_event=False) - max_connecting = 1 if self._backoff else self._max_connecting - while not (self.conns or self._pending < max_connecting): + while not (self.conns or self._pending < self.max_connecting): timeout = deadline - time.monotonic() if deadline else None if not _cond_wait(self._max_connecting_cond, timeout): # Timed out, notify the next thread to ensure a # timeout doesn't consume the condition. - if self.conns or self._pending < max_connecting: + if self.conns or self._pending < self.max_connecting: self._max_connecting_cond.notify() emitted_event = True self._raise_wait_queue_timeout(checkout_started_time) diff --git a/test/asynchronous/test_pooling.py b/test/asynchronous/test_pooling.py index 9009577f1c..b2e9d9d2b7 100644 --- a/test/asynchronous/test_pooling.py +++ b/test/asynchronous/test_pooling.py @@ -395,20 +395,16 @@ async def test_checkout_more_than_max_pool_size(self): await asyncio.sleep(0.05) await pool.close() - async def _check_maxConnecting( - self, client: AsyncMongoClient, backoff=False - ) -> tuple[int, int]: - coll = client.test.test.with_options(read_preference=ReadPreference.PRIMARY) - await coll.insert_one({}) - + async def test_maxConnecting(self): + client = await self.async_rs_or_single_client() + await self.client.test.test.insert_one({}) + self.addAsyncCleanup(self.client.test.test.delete_many, {}) pool = await async_get_pool(client) - if backoff: - pool._backoff = 1 docs = [] # Run 50 short running operations async def find_one(): - docs.append(await coll.find_one({})) + docs.append(await client.test.test.find_one({})) tasks = [ConcurrentRunner(target=find_one) for _ in range(50)] for task in tasks: @@ -416,23 +412,15 @@ async def find_one(): for task in tasks: await task.join(10) - await coll.delete_many({}) - - return len(docs), len(pool.conns) - - async def test_maxConnecting(self): - client = await self.async_rs_or_single_client() - num_docs, num_conns = await self._check_maxConnecting(client) - - self.assertEqual(num_docs, 50) - self.assertLessEqual(num_conns, 50) + self.assertEqual(len(docs), 50) + self.assertLessEqual(len(pool.conns), 50) # TLS and auth make connection establishment more expensive than # the query which leads to more threads hitting maxConnecting. # The end result is fewer total connections and better latency. if async_client_context.tls and async_client_context.auth_enabled: - self.assertLessEqual(num_conns, 30) + self.assertLessEqual(len(pool.conns), 30) else: - self.assertLessEqual(num_conns, 50) + self.assertLessEqual(len(pool.conns), 50) # MongoDB 4.4.1 with auth + ssl: # maxConnecting = 2: 6 connections in ~0.231+ seconds # maxConnecting = unbounded: 50 connections in ~0.642+ seconds @@ -440,7 +428,7 @@ async def test_maxConnecting(self): # MongoDB 4.4.1 with no-auth no-ssl Python 3.8: # maxConnecting = 2: 15-22 connections in ~0.108+ seconds # maxConnecting = unbounded: 30+ connections in ~0.140+ seconds - print(num_conns) + print(len(pool.conns)) @async_client_context.require_failCommand_appName async def test_csot_timeout_message(self): @@ -588,17 +576,13 @@ async def test_pool_backoff_preserves_existing_connections(self): await pool.close() async def test_pool_backoff_limits_maxConnecting(self): - client = await self.async_rs_or_single_client(maxConnecting=30) - _, baseline_conns = await self._check_maxConnecting(client) - await client.close() - - client = await self.async_rs_or_single_client(maxConnecting=30) - _, backoff_conns = await self._check_maxConnecting(client, backoff=True) + client = await self.async_rs_or_single_client(maxConnecting=10) + pool = await async_get_pool(client) + assert pool.maxConnecting == 10 + pool._backoff = 1 + assert pool.maxConnecting == 1 await client.close() - # We should have created less conns due to limiting maxConnecting. - self.assertLess(backoff_conns, baseline_conns) - class TestPoolMaxSize(_TestPoolingBase): async def test_max_pool_size(self): diff --git a/test/test_pooling.py b/test/test_pooling.py index c9dc6dffc7..244582a86b 100644 --- a/test/test_pooling.py +++ b/test/test_pooling.py @@ -395,18 +395,16 @@ def test_checkout_more_than_max_pool_size(self): time.sleep(0.05) pool.close() - def _check_maxConnecting(self, client: MongoClient, backoff=False) -> tuple[int, int]: - coll = client.test.test.with_options(read_preference=ReadPreference.PRIMARY) - coll.insert_one({}) - + def test_maxConnecting(self): + client = self.rs_or_single_client() + self.client.test.test.insert_one({}) + self.addCleanup(self.client.test.test.delete_many, {}) pool = get_pool(client) - if backoff: - pool._backoff = 1 docs = [] # Run 50 short running operations def find_one(): - docs.append(coll.find_one({})) + docs.append(client.test.test.find_one({})) tasks = [ConcurrentRunner(target=find_one) for _ in range(50)] for task in tasks: @@ -414,23 +412,15 @@ def find_one(): for task in tasks: task.join(10) - coll.delete_many({}) - - return len(docs), len(pool.conns) - - def test_maxConnecting(self): - client = self.rs_or_single_client() - num_docs, num_conns = self._check_maxConnecting(client) - - self.assertEqual(num_docs, 50) - self.assertLessEqual(num_conns, 50) + self.assertEqual(len(docs), 50) + self.assertLessEqual(len(pool.conns), 50) # TLS and auth make connection establishment more expensive than # the query which leads to more threads hitting maxConnecting. # The end result is fewer total connections and better latency. if client_context.tls and client_context.auth_enabled: - self.assertLessEqual(num_conns, 30) + self.assertLessEqual(len(pool.conns), 30) else: - self.assertLessEqual(num_conns, 50) + self.assertLessEqual(len(pool.conns), 50) # MongoDB 4.4.1 with auth + ssl: # maxConnecting = 2: 6 connections in ~0.231+ seconds # maxConnecting = unbounded: 50 connections in ~0.642+ seconds @@ -438,7 +428,7 @@ def test_maxConnecting(self): # MongoDB 4.4.1 with no-auth no-ssl Python 3.8: # maxConnecting = 2: 15-22 connections in ~0.108+ seconds # maxConnecting = unbounded: 30+ connections in ~0.140+ seconds - print(num_conns) + print(len(pool.conns)) @client_context.require_failCommand_appName def test_csot_timeout_message(self): @@ -584,17 +574,13 @@ def test_pool_backoff_preserves_existing_connections(self): pool.close() def test_pool_backoff_limits_maxConnecting(self): - client = self.rs_or_single_client(maxConnecting=30) - _, baseline_conns = self._check_maxConnecting(client) - client.close() - - client = self.rs_or_single_client(maxConnecting=30) - _, backoff_conns = self._check_maxConnecting(client, backoff=True) + client = self.rs_or_single_client(maxConnecting=10) + pool = get_pool(client) + assert pool.maxConnecting == 10 + pool._backoff = 1 + assert pool.maxConnecting == 1 client.close() - # We should have created less conns due to limiting maxConnecting. - self.assertLess(backoff_conns, baseline_conns) - class TestPoolMaxSize(_TestPoolingBase): def test_max_pool_size(self): From 64aa0afc7ee5d588b24df772fc5b63656eab8de6 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 5 Sep 2025 15:59:45 -0500 Subject: [PATCH 43/53] update test --- test/asynchronous/test_pooling.py | 2 ++ test/test_pooling.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/test/asynchronous/test_pooling.py b/test/asynchronous/test_pooling.py index b2e9d9d2b7..ddcbad6f54 100644 --- a/test/asynchronous/test_pooling.py +++ b/test/asynchronous/test_pooling.py @@ -581,6 +581,8 @@ async def test_pool_backoff_limits_maxConnecting(self): assert pool.maxConnecting == 10 pool._backoff = 1 assert pool.maxConnecting == 1 + pool._backoff = 0 + assert pool.maxConnecting == 10 await client.close() diff --git a/test/test_pooling.py b/test/test_pooling.py index 244582a86b..e045f7bb77 100644 --- a/test/test_pooling.py +++ b/test/test_pooling.py @@ -579,6 +579,8 @@ def test_pool_backoff_limits_maxConnecting(self): assert pool.maxConnecting == 10 pool._backoff = 1 assert pool.maxConnecting == 1 + pool._backoff = 0 + assert pool.maxConnecting == 10 client.close() From 7f6335e7a000e2fb9624f4060aaa3cddaaadd680 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 10 Sep 2025 08:36:03 -0500 Subject: [PATCH 44/53] fix test --- test/asynchronous/test_pooling.py | 6 +++--- test/test_pooling.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/test/asynchronous/test_pooling.py b/test/asynchronous/test_pooling.py index ddcbad6f54..6cbdf7a65c 100644 --- a/test/asynchronous/test_pooling.py +++ b/test/asynchronous/test_pooling.py @@ -578,11 +578,11 @@ async def test_pool_backoff_preserves_existing_connections(self): async def test_pool_backoff_limits_maxConnecting(self): client = await self.async_rs_or_single_client(maxConnecting=10) pool = await async_get_pool(client) - assert pool.maxConnecting == 10 + assert pool.max_connecting == 10 pool._backoff = 1 - assert pool.maxConnecting == 1 + assert pool.max_connecting == 1 pool._backoff = 0 - assert pool.maxConnecting == 10 + assert pool.max_connecting == 10 await client.close() diff --git a/test/test_pooling.py b/test/test_pooling.py index e045f7bb77..f3bfcf4ba2 100644 --- a/test/test_pooling.py +++ b/test/test_pooling.py @@ -576,11 +576,11 @@ def test_pool_backoff_preserves_existing_connections(self): def test_pool_backoff_limits_maxConnecting(self): client = self.rs_or_single_client(maxConnecting=10) pool = get_pool(client) - assert pool.maxConnecting == 10 + assert pool.max_connecting == 10 pool._backoff = 1 - assert pool.maxConnecting == 1 + assert pool.max_connecting == 1 pool._backoff = 0 - assert pool.maxConnecting == 10 + assert pool.max_connecting == 10 client.close() From 6db793dbe8f70f5e737e6bcc8aace39caf01c58c Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 10 Sep 2025 08:38:46 -0500 Subject: [PATCH 45/53] undo lock file changes --- uv.lock | 98 ++++++++++++++++++++++++--------------------------------- 1 file changed, 41 insertions(+), 57 deletions(-) diff --git a/uv.lock b/uv.lock index 9c74220460..9c45c4cdb9 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 3 +revision = 2 requires-python = ">=3.9" resolution-markers = [ "python_full_version == '3.14.*'", @@ -1047,53 +1047,46 @@ dependencies = [ [[package]] name = "mypy" -version = "1.17.1" +version = "1.14.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "mypy-extensions" }, - { name = "pathspec" }, { name = "tomli", marker = "python_full_version < '3.11'" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/8e/22/ea637422dedf0bf36f3ef238eab4e455e2a0dcc3082b5cc067615347ab8e/mypy-1.17.1.tar.gz", hash = "sha256:25e01ec741ab5bb3eec8ba9cdb0f769230368a22c959c4937360efb89b7e9f01", size = 3352570, upload-time = "2025-07-31T07:54:19.204Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/77/a9/3d7aa83955617cdf02f94e50aab5c830d205cfa4320cf124ff64acce3a8e/mypy-1.17.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3fbe6d5555bf608c47203baa3e72dbc6ec9965b3d7c318aa9a4ca76f465bd972", size = 11003299, upload-time = "2025-07-31T07:54:06.425Z" }, - { url = "https://files.pythonhosted.org/packages/83/e8/72e62ff837dd5caaac2b4a5c07ce769c8e808a00a65e5d8f94ea9c6f20ab/mypy-1.17.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:80ef5c058b7bce08c83cac668158cb7edea692e458d21098c7d3bce35a5d43e7", size = 10125451, upload-time = "2025-07-31T07:53:52.974Z" }, - { url = "https://files.pythonhosted.org/packages/7d/10/f3f3543f6448db11881776f26a0ed079865926b0c841818ee22de2c6bbab/mypy-1.17.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c4a580f8a70c69e4a75587bd925d298434057fe2a428faaf927ffe6e4b9a98df", size = 11916211, upload-time = "2025-07-31T07:53:18.879Z" }, - { url = "https://files.pythonhosted.org/packages/06/bf/63e83ed551282d67bb3f7fea2cd5561b08d2bb6eb287c096539feb5ddbc5/mypy-1.17.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dd86bb649299f09d987a2eebb4d52d10603224500792e1bee18303bbcc1ce390", size = 12652687, upload-time = "2025-07-31T07:53:30.544Z" }, - { url = "https://files.pythonhosted.org/packages/69/66/68f2eeef11facf597143e85b694a161868b3b006a5fbad50e09ea117ef24/mypy-1.17.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:a76906f26bd8d51ea9504966a9c25419f2e668f012e0bdf3da4ea1526c534d94", size = 12896322, upload-time = "2025-07-31T07:53:50.74Z" }, - { url = "https://files.pythonhosted.org/packages/a3/87/8e3e9c2c8bd0d7e071a89c71be28ad088aaecbadf0454f46a540bda7bca6/mypy-1.17.1-cp310-cp310-win_amd64.whl", hash = "sha256:e79311f2d904ccb59787477b7bd5d26f3347789c06fcd7656fa500875290264b", size = 9507962, upload-time = "2025-07-31T07:53:08.431Z" }, - { url = "https://files.pythonhosted.org/packages/46/cf/eadc80c4e0a70db1c08921dcc220357ba8ab2faecb4392e3cebeb10edbfa/mypy-1.17.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ad37544be07c5d7fba814eb370e006df58fed8ad1ef33ed1649cb1889ba6ff58", size = 10921009, upload-time = "2025-07-31T07:53:23.037Z" }, - { url = "https://files.pythonhosted.org/packages/5d/c1/c869d8c067829ad30d9bdae051046561552516cfb3a14f7f0347b7d973ee/mypy-1.17.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:064e2ff508e5464b4bd807a7c1625bc5047c5022b85c70f030680e18f37273a5", size = 10047482, upload-time = "2025-07-31T07:53:26.151Z" }, - { url = "https://files.pythonhosted.org/packages/98/b9/803672bab3fe03cee2e14786ca056efda4bb511ea02dadcedde6176d06d0/mypy-1.17.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:70401bbabd2fa1aa7c43bb358f54037baf0586f41e83b0ae67dd0534fc64edfd", size = 11832883, upload-time = "2025-07-31T07:53:47.948Z" }, - { url = "https://files.pythonhosted.org/packages/88/fb/fcdac695beca66800918c18697b48833a9a6701de288452b6715a98cfee1/mypy-1.17.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e92bdc656b7757c438660f775f872a669b8ff374edc4d18277d86b63edba6b8b", size = 12566215, upload-time = "2025-07-31T07:54:04.031Z" }, - { url = "https://files.pythonhosted.org/packages/7f/37/a932da3d3dace99ee8eb2043b6ab03b6768c36eb29a02f98f46c18c0da0e/mypy-1.17.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c1fdf4abb29ed1cb091cf432979e162c208a5ac676ce35010373ff29247bcad5", size = 12751956, upload-time = "2025-07-31T07:53:36.263Z" }, - { url = "https://files.pythonhosted.org/packages/8c/cf/6438a429e0f2f5cab8bc83e53dbebfa666476f40ee322e13cac5e64b79e7/mypy-1.17.1-cp311-cp311-win_amd64.whl", hash = "sha256:ff2933428516ab63f961644bc49bc4cbe42bbffb2cd3b71cc7277c07d16b1a8b", size = 9507307, upload-time = "2025-07-31T07:53:59.734Z" }, - { url = "https://files.pythonhosted.org/packages/17/a2/7034d0d61af8098ec47902108553122baa0f438df8a713be860f7407c9e6/mypy-1.17.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:69e83ea6553a3ba79c08c6e15dbd9bfa912ec1e493bf75489ef93beb65209aeb", size = 11086295, upload-time = "2025-07-31T07:53:28.124Z" }, - { url = "https://files.pythonhosted.org/packages/14/1f/19e7e44b594d4b12f6ba8064dbe136505cec813549ca3e5191e40b1d3cc2/mypy-1.17.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1b16708a66d38abb1e6b5702f5c2c87e133289da36f6a1d15f6a5221085c6403", size = 10112355, upload-time = "2025-07-31T07:53:21.121Z" }, - { url = "https://files.pythonhosted.org/packages/5b/69/baa33927e29e6b4c55d798a9d44db5d394072eef2bdc18c3e2048c9ed1e9/mypy-1.17.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:89e972c0035e9e05823907ad5398c5a73b9f47a002b22359b177d40bdaee7056", size = 11875285, upload-time = "2025-07-31T07:53:55.293Z" }, - { url = "https://files.pythonhosted.org/packages/90/13/f3a89c76b0a41e19490b01e7069713a30949d9a6c147289ee1521bcea245/mypy-1.17.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:03b6d0ed2b188e35ee6d5c36b5580cffd6da23319991c49ab5556c023ccf1341", size = 12737895, upload-time = "2025-07-31T07:53:43.623Z" }, - { url = "https://files.pythonhosted.org/packages/23/a1/c4ee79ac484241301564072e6476c5a5be2590bc2e7bfd28220033d2ef8f/mypy-1.17.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c837b896b37cd103570d776bda106eabb8737aa6dd4f248451aecf53030cdbeb", size = 12931025, upload-time = "2025-07-31T07:54:17.125Z" }, - { url = "https://files.pythonhosted.org/packages/89/b8/7409477be7919a0608900e6320b155c72caab4fef46427c5cc75f85edadd/mypy-1.17.1-cp312-cp312-win_amd64.whl", hash = "sha256:665afab0963a4b39dff7c1fa563cc8b11ecff7910206db4b2e64dd1ba25aed19", size = 9584664, upload-time = "2025-07-31T07:54:12.842Z" }, - { url = "https://files.pythonhosted.org/packages/5b/82/aec2fc9b9b149f372850291827537a508d6c4d3664b1750a324b91f71355/mypy-1.17.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:93378d3203a5c0800c6b6d850ad2f19f7a3cdf1a3701d3416dbf128805c6a6a7", size = 11075338, upload-time = "2025-07-31T07:53:38.873Z" }, - { url = "https://files.pythonhosted.org/packages/07/ac/ee93fbde9d2242657128af8c86f5d917cd2887584cf948a8e3663d0cd737/mypy-1.17.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:15d54056f7fe7a826d897789f53dd6377ec2ea8ba6f776dc83c2902b899fee81", size = 10113066, upload-time = "2025-07-31T07:54:14.707Z" }, - { url = "https://files.pythonhosted.org/packages/5a/68/946a1e0be93f17f7caa56c45844ec691ca153ee8b62f21eddda336a2d203/mypy-1.17.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:209a58fed9987eccc20f2ca94afe7257a8f46eb5df1fb69958650973230f91e6", size = 11875473, upload-time = "2025-07-31T07:53:14.504Z" }, - { url = "https://files.pythonhosted.org/packages/9f/0f/478b4dce1cb4f43cf0f0d00fba3030b21ca04a01b74d1cd272a528cf446f/mypy-1.17.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:099b9a5da47de9e2cb5165e581f158e854d9e19d2e96b6698c0d64de911dd849", size = 12744296, upload-time = "2025-07-31T07:53:03.896Z" }, - { url = "https://files.pythonhosted.org/packages/ca/70/afa5850176379d1b303f992a828de95fc14487429a7139a4e0bdd17a8279/mypy-1.17.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:fa6ffadfbe6994d724c5a1bb6123a7d27dd68fc9c059561cd33b664a79578e14", size = 12914657, upload-time = "2025-07-31T07:54:08.576Z" }, - { url = "https://files.pythonhosted.org/packages/53/f9/4a83e1c856a3d9c8f6edaa4749a4864ee98486e9b9dbfbc93842891029c2/mypy-1.17.1-cp313-cp313-win_amd64.whl", hash = "sha256:9a2b7d9180aed171f033c9f2fc6c204c1245cf60b0cb61cf2e7acc24eea78e0a", size = 9593320, upload-time = "2025-07-31T07:53:01.341Z" }, - { url = "https://files.pythonhosted.org/packages/38/56/79c2fac86da57c7d8c48622a05873eaab40b905096c33597462713f5af90/mypy-1.17.1-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:15a83369400454c41ed3a118e0cc58bd8123921a602f385cb6d6ea5df050c733", size = 11040037, upload-time = "2025-07-31T07:54:10.942Z" }, - { url = "https://files.pythonhosted.org/packages/4d/c3/adabe6ff53638e3cad19e3547268482408323b1e68bf082c9119000cd049/mypy-1.17.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:55b918670f692fc9fba55c3298d8a3beae295c5cded0a55dccdc5bbead814acd", size = 10131550, upload-time = "2025-07-31T07:53:41.307Z" }, - { url = "https://files.pythonhosted.org/packages/b8/c5/2e234c22c3bdeb23a7817af57a58865a39753bde52c74e2c661ee0cfc640/mypy-1.17.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:62761474061feef6f720149d7ba876122007ddc64adff5ba6f374fda35a018a0", size = 11872963, upload-time = "2025-07-31T07:53:16.878Z" }, - { url = "https://files.pythonhosted.org/packages/ab/26/c13c130f35ca8caa5f2ceab68a247775648fdcd6c9a18f158825f2bc2410/mypy-1.17.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c49562d3d908fd49ed0938e5423daed8d407774a479b595b143a3d7f87cdae6a", size = 12710189, upload-time = "2025-07-31T07:54:01.962Z" }, - { url = "https://files.pythonhosted.org/packages/82/df/c7d79d09f6de8383fe800521d066d877e54d30b4fb94281c262be2df84ef/mypy-1.17.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:397fba5d7616a5bc60b45c7ed204717eaddc38f826e3645402c426057ead9a91", size = 12900322, upload-time = "2025-07-31T07:53:10.551Z" }, - { url = "https://files.pythonhosted.org/packages/b8/98/3d5a48978b4f708c55ae832619addc66d677f6dc59f3ebad71bae8285ca6/mypy-1.17.1-cp314-cp314-win_amd64.whl", hash = "sha256:9d6b20b97d373f41617bd0708fd46aa656059af57f2ef72aa8c7d6a2b73b74ed", size = 9751879, upload-time = "2025-07-31T07:52:56.683Z" }, - { url = "https://files.pythonhosted.org/packages/29/cb/673e3d34e5d8de60b3a61f44f80150a738bff568cd6b7efb55742a605e98/mypy-1.17.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5d1092694f166a7e56c805caaf794e0585cabdbf1df36911c414e4e9abb62ae9", size = 10992466, upload-time = "2025-07-31T07:53:57.574Z" }, - { url = "https://files.pythonhosted.org/packages/0c/d0/fe1895836eea3a33ab801561987a10569df92f2d3d4715abf2cfeaa29cb2/mypy-1.17.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:79d44f9bfb004941ebb0abe8eff6504223a9c1ac51ef967d1263c6572bbebc99", size = 10117638, upload-time = "2025-07-31T07:53:34.256Z" }, - { url = "https://files.pythonhosted.org/packages/97/f3/514aa5532303aafb95b9ca400a31054a2bd9489de166558c2baaeea9c522/mypy-1.17.1-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b01586eed696ec905e61bd2568f48740f7ac4a45b3a468e6423a03d3788a51a8", size = 11915673, upload-time = "2025-07-31T07:52:59.361Z" }, - { url = "https://files.pythonhosted.org/packages/ab/c3/c0805f0edec96fe8e2c048b03769a6291523d509be8ee7f56ae922fa3882/mypy-1.17.1-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:43808d9476c36b927fbcd0b0255ce75efe1b68a080154a38ae68a7e62de8f0f8", size = 12649022, upload-time = "2025-07-31T07:53:45.92Z" }, - { url = "https://files.pythonhosted.org/packages/45/3e/d646b5a298ada21a8512fa7e5531f664535a495efa672601702398cea2b4/mypy-1.17.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:feb8cc32d319edd5859da2cc084493b3e2ce5e49a946377663cc90f6c15fb259", size = 12895536, upload-time = "2025-07-31T07:53:06.17Z" }, - { url = "https://files.pythonhosted.org/packages/14/55/e13d0dcd276975927d1f4e9e2ec4fd409e199f01bdc671717e673cc63a22/mypy-1.17.1-cp39-cp39-win_amd64.whl", hash = "sha256:d7598cf74c3e16539d4e2f0b8d8c318e00041553d83d4861f87c7a72e95ac24d", size = 9512564, upload-time = "2025-07-31T07:53:12.346Z" }, - { url = "https://files.pythonhosted.org/packages/1d/f3/8fcd2af0f5b806f6cf463efaffd3c9548a28f84220493ecd38d127b6b66d/mypy-1.17.1-py3-none-any.whl", hash = "sha256:a9f52c0351c21fe24c21d8c0eb1f62967b262d6729393397b6f443c3b773c3b9", size = 2283411, upload-time = "2025-07-31T07:53:24.664Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/b9/eb/2c92d8ea1e684440f54fa49ac5d9a5f19967b7b472a281f419e69a8d228e/mypy-1.14.1.tar.gz", hash = "sha256:7ec88144fe9b510e8475ec2f5f251992690fcf89ccb4500b214b4226abcd32d6", size = 3216051, upload-time = "2024-12-30T16:39:07.335Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9b/7a/87ae2adb31d68402da6da1e5f30c07ea6063e9f09b5e7cfc9dfa44075e74/mypy-1.14.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:52686e37cf13d559f668aa398dd7ddf1f92c5d613e4f8cb262be2fb4fedb0fcb", size = 11211002, upload-time = "2024-12-30T16:37:22.435Z" }, + { url = "https://files.pythonhosted.org/packages/e1/23/eada4c38608b444618a132be0d199b280049ded278b24cbb9d3fc59658e4/mypy-1.14.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:1fb545ca340537d4b45d3eecdb3def05e913299ca72c290326be19b3804b39c0", size = 10358400, upload-time = "2024-12-30T16:37:53.526Z" }, + { url = "https://files.pythonhosted.org/packages/43/c9/d6785c6f66241c62fd2992b05057f404237deaad1566545e9f144ced07f5/mypy-1.14.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:90716d8b2d1f4cd503309788e51366f07c56635a3309b0f6a32547eaaa36a64d", size = 12095172, upload-time = "2024-12-30T16:37:50.332Z" }, + { url = "https://files.pythonhosted.org/packages/c3/62/daa7e787770c83c52ce2aaf1a111eae5893de9e004743f51bfcad9e487ec/mypy-1.14.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2ae753f5c9fef278bcf12e1a564351764f2a6da579d4a81347e1d5a15819997b", size = 12828732, upload-time = "2024-12-30T16:37:29.96Z" }, + { url = "https://files.pythonhosted.org/packages/1b/a2/5fb18318a3637f29f16f4e41340b795da14f4751ef4f51c99ff39ab62e52/mypy-1.14.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:e0fe0f5feaafcb04505bcf439e991c6d8f1bf8b15f12b05feeed96e9e7bf1427", size = 13012197, upload-time = "2024-12-30T16:38:05.037Z" }, + { url = "https://files.pythonhosted.org/packages/28/99/e153ce39105d164b5f02c06c35c7ba958aaff50a2babba7d080988b03fe7/mypy-1.14.1-cp310-cp310-win_amd64.whl", hash = "sha256:7d54bd85b925e501c555a3227f3ec0cfc54ee8b6930bd6141ec872d1c572f81f", size = 9780836, upload-time = "2024-12-30T16:37:19.726Z" }, + { url = "https://files.pythonhosted.org/packages/da/11/a9422850fd506edbcdc7f6090682ecceaf1f87b9dd847f9df79942da8506/mypy-1.14.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f995e511de847791c3b11ed90084a7a0aafdc074ab88c5a9711622fe4751138c", size = 11120432, upload-time = "2024-12-30T16:37:11.533Z" }, + { url = "https://files.pythonhosted.org/packages/b6/9e/47e450fd39078d9c02d620545b2cb37993a8a8bdf7db3652ace2f80521ca/mypy-1.14.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d64169ec3b8461311f8ce2fd2eb5d33e2d0f2c7b49116259c51d0d96edee48d1", size = 10279515, upload-time = "2024-12-30T16:37:40.724Z" }, + { url = "https://files.pythonhosted.org/packages/01/b5/6c8d33bd0f851a7692a8bfe4ee75eb82b6983a3cf39e5e32a5d2a723f0c1/mypy-1.14.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ba24549de7b89b6381b91fbc068d798192b1b5201987070319889e93038967a8", size = 12025791, upload-time = "2024-12-30T16:36:58.73Z" }, + { url = "https://files.pythonhosted.org/packages/f0/4c/e10e2c46ea37cab5c471d0ddaaa9a434dc1d28650078ac1b56c2d7b9b2e4/mypy-1.14.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:183cf0a45457d28ff9d758730cd0210419ac27d4d3f285beda038c9083363b1f", size = 12749203, upload-time = "2024-12-30T16:37:03.741Z" }, + { url = "https://files.pythonhosted.org/packages/88/55/beacb0c69beab2153a0f57671ec07861d27d735a0faff135a494cd4f5020/mypy-1.14.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f2a0ecc86378f45347f586e4163d1769dd81c5a223d577fe351f26b179e148b1", size = 12885900, upload-time = "2024-12-30T16:37:57.948Z" }, + { url = "https://files.pythonhosted.org/packages/a2/75/8c93ff7f315c4d086a2dfcde02f713004357d70a163eddb6c56a6a5eff40/mypy-1.14.1-cp311-cp311-win_amd64.whl", hash = "sha256:ad3301ebebec9e8ee7135d8e3109ca76c23752bac1e717bc84cd3836b4bf3eae", size = 9777869, upload-time = "2024-12-30T16:37:33.428Z" }, + { url = "https://files.pythonhosted.org/packages/43/1b/b38c079609bb4627905b74fc6a49849835acf68547ac33d8ceb707de5f52/mypy-1.14.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:30ff5ef8519bbc2e18b3b54521ec319513a26f1bba19a7582e7b1f58a6e69f14", size = 11266668, upload-time = "2024-12-30T16:38:02.211Z" }, + { url = "https://files.pythonhosted.org/packages/6b/75/2ed0d2964c1ffc9971c729f7a544e9cd34b2cdabbe2d11afd148d7838aa2/mypy-1.14.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:cb9f255c18052343c70234907e2e532bc7e55a62565d64536dbc7706a20b78b9", size = 10254060, upload-time = "2024-12-30T16:37:46.131Z" }, + { url = "https://files.pythonhosted.org/packages/a1/5f/7b8051552d4da3c51bbe8fcafffd76a6823779101a2b198d80886cd8f08e/mypy-1.14.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8b4e3413e0bddea671012b063e27591b953d653209e7a4fa5e48759cda77ca11", size = 11933167, upload-time = "2024-12-30T16:37:43.534Z" }, + { url = "https://files.pythonhosted.org/packages/04/90/f53971d3ac39d8b68bbaab9a4c6c58c8caa4d5fd3d587d16f5927eeeabe1/mypy-1.14.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:553c293b1fbdebb6c3c4030589dab9fafb6dfa768995a453d8a5d3b23784af2e", size = 12864341, upload-time = "2024-12-30T16:37:36.249Z" }, + { url = "https://files.pythonhosted.org/packages/03/d2/8bc0aeaaf2e88c977db41583559319f1821c069e943ada2701e86d0430b7/mypy-1.14.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fad79bfe3b65fe6a1efaed97b445c3d37f7be9fdc348bdb2d7cac75579607c89", size = 12972991, upload-time = "2024-12-30T16:37:06.743Z" }, + { url = "https://files.pythonhosted.org/packages/6f/17/07815114b903b49b0f2cf7499f1c130e5aa459411596668267535fe9243c/mypy-1.14.1-cp312-cp312-win_amd64.whl", hash = "sha256:8fa2220e54d2946e94ab6dbb3ba0a992795bd68b16dc852db33028df2b00191b", size = 9879016, upload-time = "2024-12-30T16:37:15.02Z" }, + { url = "https://files.pythonhosted.org/packages/9e/15/bb6a686901f59222275ab228453de741185f9d54fecbaacec041679496c6/mypy-1.14.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:92c3ed5afb06c3a8e188cb5da4984cab9ec9a77ba956ee419c68a388b4595255", size = 11252097, upload-time = "2024-12-30T16:37:25.144Z" }, + { url = "https://files.pythonhosted.org/packages/f8/b3/8b0f74dfd072c802b7fa368829defdf3ee1566ba74c32a2cb2403f68024c/mypy-1.14.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:dbec574648b3e25f43d23577309b16534431db4ddc09fda50841f1e34e64ed34", size = 10239728, upload-time = "2024-12-30T16:38:08.634Z" }, + { url = "https://files.pythonhosted.org/packages/c5/9b/4fd95ab20c52bb5b8c03cc49169be5905d931de17edfe4d9d2986800b52e/mypy-1.14.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8c6d94b16d62eb3e947281aa7347d78236688e21081f11de976376cf010eb31a", size = 11924965, upload-time = "2024-12-30T16:38:12.132Z" }, + { url = "https://files.pythonhosted.org/packages/56/9d/4a236b9c57f5d8f08ed346914b3f091a62dd7e19336b2b2a0d85485f82ff/mypy-1.14.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d4b19b03fdf54f3c5b2fa474c56b4c13c9dbfb9a2db4370ede7ec11a2c5927d9", size = 12867660, upload-time = "2024-12-30T16:38:17.342Z" }, + { url = "https://files.pythonhosted.org/packages/40/88/a61a5497e2f68d9027de2bb139c7bb9abaeb1be1584649fa9d807f80a338/mypy-1.14.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:0c911fde686394753fff899c409fd4e16e9b294c24bfd5e1ea4675deae1ac6fd", size = 12969198, upload-time = "2024-12-30T16:38:32.839Z" }, + { url = "https://files.pythonhosted.org/packages/54/da/3d6fc5d92d324701b0c23fb413c853892bfe0e1dbe06c9138037d459756b/mypy-1.14.1-cp313-cp313-win_amd64.whl", hash = "sha256:8b21525cb51671219f5307be85f7e646a153e5acc656e5cebf64bfa076c50107", size = 9885276, upload-time = "2024-12-30T16:38:20.828Z" }, + { url = "https://files.pythonhosted.org/packages/ca/1f/186d133ae2514633f8558e78cd658070ba686c0e9275c5a5c24a1e1f0d67/mypy-1.14.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:3888a1816d69f7ab92092f785a462944b3ca16d7c470d564165fe703b0970c35", size = 11200493, upload-time = "2024-12-30T16:38:26.935Z" }, + { url = "https://files.pythonhosted.org/packages/af/fc/4842485d034e38a4646cccd1369f6b1ccd7bc86989c52770d75d719a9941/mypy-1.14.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:46c756a444117c43ee984bd055db99e498bc613a70bbbc120272bd13ca579fbc", size = 10357702, upload-time = "2024-12-30T16:38:50.623Z" }, + { url = "https://files.pythonhosted.org/packages/b4/e6/457b83f2d701e23869cfec013a48a12638f75b9d37612a9ddf99072c1051/mypy-1.14.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:27fc248022907e72abfd8e22ab1f10e903915ff69961174784a3900a8cba9ad9", size = 12091104, upload-time = "2024-12-30T16:38:53.735Z" }, + { url = "https://files.pythonhosted.org/packages/f1/bf/76a569158db678fee59f4fd30b8e7a0d75bcbaeef49edd882a0d63af6d66/mypy-1.14.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:499d6a72fb7e5de92218db961f1a66d5f11783f9ae549d214617edab5d4dbdbb", size = 12830167, upload-time = "2024-12-30T16:38:56.437Z" }, + { url = "https://files.pythonhosted.org/packages/43/bc/0bc6b694b3103de9fed61867f1c8bd33336b913d16831431e7cb48ef1c92/mypy-1.14.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:57961db9795eb566dc1d1b4e9139ebc4c6b0cb6e7254ecde69d1552bf7613f60", size = 13013834, upload-time = "2024-12-30T16:38:59.204Z" }, + { url = "https://files.pythonhosted.org/packages/b0/79/5f5ec47849b6df1e6943d5fd8e6632fbfc04b4fd4acfa5a5a9535d11b4e2/mypy-1.14.1-cp39-cp39-win_amd64.whl", hash = "sha256:07ba89fdcc9451f2ebb02853deb6aaaa3d2239a236669a63ab3801bbf923ef5c", size = 9781231, upload-time = "2024-12-30T16:39:05.124Z" }, + { url = "https://files.pythonhosted.org/packages/a0/b5/32dd67b69a16d088e533962e5044e51004176a9952419de0370cdaead0f8/mypy-1.14.1-py3-none-any.whl", hash = "sha256:b66a60cc4073aeb8ae00057f9c1f64d49e90f918fbcef9a977eb121da8b8f1d1", size = 2752905, upload-time = "2024-12-30T16:38:42.021Z" }, ] [[package]] @@ -1123,15 +1116,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484", size = 66469, upload-time = "2025-04-19T11:48:57.875Z" }, ] -[[package]] -name = "pathspec" -version = "0.12.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/ca/bc/f35b8446f4531a7cb215605d100cd88b7ac6f44ab3fc94870c120ab3adbf/pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712", size = 51043, upload-time = "2023-12-10T22:30:45Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/cc/20/ff623b09d963f88bfde16306a54e12ee5ea43e9b597108672ff3a408aad6/pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08", size = 31191, upload-time = "2023-12-10T22:30:43.14Z" }, -] - [[package]] name = "pip" version = "25.2" @@ -1331,7 +1315,7 @@ provides-extras = ["aws", "docs", "encryption", "gssapi", "ocsp", "snappy", "tes [package.metadata.requires-dev] coverage = [ - { name = "coverage", specifier = ">=5,<=7.10.3" }, + { name = "coverage", specifier = ">=5,<=7.5" }, { name = "pytest-cov" }, ] dev = [{ name = "pre-commit", specifier = ">=4.0" }] @@ -1345,9 +1329,9 @@ perf = [{ name = "simplejson" }] pip = [{ name = "pip" }] pymongocrypt-source = [{ name = "pymongocrypt", git = "https://github.com/mongodb/libmongocrypt?subdirectory=bindings%2Fpython&rev=master" }] typing = [ - { name = "mypy", specifier = "==1.17.1" }, + { name = "mypy", specifier = "==1.14.1" }, { name = "pip" }, - { name = "pyright", specifier = "==1.1.403" }, + { name = "pyright", specifier = "==1.1.392.post0" }, { name = "typing-extensions" }, ] @@ -1391,15 +1375,15 @@ wheels = [ [[package]] name = "pyright" -version = "1.1.403" +version = "1.1.392.post0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "nodeenv" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/fe/f6/35f885264ff08c960b23d1542038d8da86971c5d8c955cfab195a4f672d7/pyright-1.1.403.tar.gz", hash = "sha256:3ab69b9f41c67fb5bbb4d7a36243256f0d549ed3608678d381d5f51863921104", size = 3913526, upload-time = "2025-07-09T07:15:52.882Z" } +sdist = { url = "https://files.pythonhosted.org/packages/66/df/3c6f6b08fba7ccf49b114dfc4bb33e25c299883fd763f93fad47ef8bc58d/pyright-1.1.392.post0.tar.gz", hash = "sha256:3b7f88de74a28dcfa90c7d90c782b6569a48c2be5f9d4add38472bdaac247ebd", size = 3789911, upload-time = "2025-01-15T15:01:20.913Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/49/b6/b04e5c2f41a5ccad74a1a4759da41adb20b4bc9d59a5e08d29ba60084d07/pyright-1.1.403-py3-none-any.whl", hash = "sha256:c0eeca5aa76cbef3fcc271259bbd785753c7ad7bcac99a9162b4c4c7daed23b3", size = 5684504, upload-time = "2025-07-09T07:15:50.958Z" }, + { url = "https://files.pythonhosted.org/packages/e7/b1/a18de17f40e4f61ca58856b9ef9b0febf74ff88978c3f7776f910071f567/pyright-1.1.392.post0-py3-none-any.whl", hash = "sha256:252f84458a46fa2f0fd4e2f91fc74f50b9ca52c757062e93f6c250c0d8329eb2", size = 5595487, upload-time = "2025-01-15T15:01:17.775Z" }, ] [[package]] From 8c2eb9157d45f244b9901cb81a95e4d9af8f7ab6 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 10 Sep 2025 10:36:14 -0500 Subject: [PATCH 46/53] PYTHON-5538 Clean up uv lock file handling (#2522) (cherry picked from commit 98e9f5ecc1396abf2fafe070de36e1bed5691930) --- justfile | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/justfile b/justfile index 24da94a499..7ac5bd33ff 100644 --- a/justfile +++ b/justfile @@ -1,10 +1,11 @@ # See https://just.systems/man/en/ for instructions set shell := ["bash", "-c"] +# Do not modify the lock file when running justfile commands. +export UV_FROZEN := "1" # Commonly used command segments. -uv_run := "uv run --frozen " -typing_run := uv_run + "--group typing --extra aws --extra encryption --extra ocsp --extra snappy --extra test --extra zstd" -docs_run := uv_run + "--extra docs" +typing_run := "uv run --group typing --extra aws --extra encryption --extra ocsp --extra snappy --extra test --extra zstd" +docs_run := "uv run --extra docs" doc_build := "./doc/_build" mypy_args := "--install-types --non-interactive" @@ -50,15 +51,15 @@ typing-pyright: && resync [group('lint')] lint: && resync - {{uv_run}} pre-commit run --all-files + uv run pre-commit run --all-files [group('lint')] lint-manual: && resync - {{uv_run}} pre-commit run --all-files --hook-stage manual + uv run pre-commit run --all-files --hook-stage manual [group('test')] test *args="-v --durations=5 --maxfail=10": && resync - {{uv_run}} --extra test pytest {{args}} + uv run --extra test pytest {{args}} [group('test')] run-tests *args: && resync From a84a1817dc6288295a3a43b9081907b3915712d9 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 10 Sep 2025 16:26:20 -0500 Subject: [PATCH 47/53] wip --- pymongo/asynchronous/pool.py | 1 + pymongo/network_layer.py | 9 +++++++++ pymongo/pool_shared.py | 1 + pymongo/synchronous/pool.py | 1 + 4 files changed, 12 insertions(+) diff --git a/pymongo/asynchronous/pool.py b/pymongo/asynchronous/pool.py index 6184cb98b7..10617333e2 100644 --- a/pymongo/asynchronous/pool.py +++ b/pymongo/asynchronous/pool.py @@ -1066,6 +1066,7 @@ async def connect(self, handler: Optional[_MongoClientErrorHandler] = None) -> A networking_interface = await _configured_protocol_interface(self.address, self.opts) # Catch KeyboardInterrupt, CancelledError, etc. and cleanup. except BaseException as error: + print("Got the TLS handshake error") # noqa: T201 async with self.lock: self.active_contexts.discard(tmp_context) if self.enabled_for_cmap: diff --git a/pymongo/network_layer.py b/pymongo/network_layer.py index 605b8dde9b..e15257c82b 100644 --- a/pymongo/network_layer.py +++ b/pymongo/network_layer.py @@ -254,6 +254,7 @@ class PyMongoBaseProtocol(Protocol): def __init__(self, timeout: Optional[float] = None): self.transport: Transport = None # type: ignore[assignment] self._timeout = timeout + self._connection_made = asyncio.get_running_loop().create_future() self._closed = asyncio.get_running_loop().create_future() self._connection_lost = False @@ -270,7 +271,13 @@ def close(self, exc: Optional[Exception] = None) -> None: self._resolve_pending(exc) self._connection_lost = True + def connection_made(self, transport: BaseTransport) -> None: + super().connection_made(transport) + self._connection_made.set_result(None) + def connection_lost(self, exc: Optional[Exception] = None) -> None: + if exc is not None and not self._connection_made.done(): + self._connection_made.set_exception(exc) self._resolve_pending(exc) if not self._closed.done(): self._closed.set_result(None) @@ -322,6 +329,7 @@ def connection_made(self, transport: BaseTransport) -> None: """ self.transport = transport # type: ignore[assignment] self.transport.set_write_buffer_limits(MAX_MESSAGE_SIZE, MAX_MESSAGE_SIZE) + super().connection_made(self) async def read(self, request_id: Optional[int], max_message_size: int) -> tuple[bytes, int]: """Read a single MongoDB Wire Protocol message from this connection.""" @@ -489,6 +497,7 @@ def connection_made(self, transport: BaseTransport) -> None: The transport argument is the transport representing the write side of the connection. """ self.transport = transport # type: ignore[assignment] + super().connection_made(self) def data_received(self, data: bytes) -> None: if self._connection_lost: diff --git a/pymongo/pool_shared.py b/pymongo/pool_shared.py index 0536dc3835..a4b4812335 100644 --- a/pymongo/pool_shared.py +++ b/pymongo/pool_shared.py @@ -278,6 +278,7 @@ async def _configured_protocol_interface( server_hostname=host, ssl=ssl_context, ) + await protocol._connection_made except _CertificateError: # Raise _CertificateError directly like we do after match_hostname # below. diff --git a/pymongo/synchronous/pool.py b/pymongo/synchronous/pool.py index 16476f5c85..dbacc63b6a 100644 --- a/pymongo/synchronous/pool.py +++ b/pymongo/synchronous/pool.py @@ -1062,6 +1062,7 @@ def connect(self, handler: Optional[_MongoClientErrorHandler] = None) -> Connect networking_interface = _configured_socket_interface(self.address, self.opts) # Catch KeyboardInterrupt, CancelledError, etc. and cleanup. except BaseException as error: + print("Got the TLS handshake error") # noqa: T201 with self.lock: self.active_contexts.discard(tmp_context) if self.enabled_for_cmap: From c5ce8dda519549044a3f9e5fc313129672a22ba8 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 11 Sep 2025 05:34:21 -0500 Subject: [PATCH 48/53] wip --- pymongo/asynchronous/pool.py | 22 ++++++++++++---------- pymongo/network_layer.py | 12 ++++-------- pymongo/pool_shared.py | 1 - pymongo/synchronous/pool.py | 22 ++++++++++++---------- 4 files changed, 28 insertions(+), 29 deletions(-) diff --git a/pymongo/asynchronous/pool.py b/pymongo/asynchronous/pool.py index 10617333e2..fd21add0d4 100644 --- a/pymongo/asynchronous/pool.py +++ b/pymongo/asynchronous/pool.py @@ -1029,6 +1029,16 @@ async def remove_stale_sockets(self, reference_generation: int) -> None: self.requests -= 1 self.size_cond.notify() + def _handle_connection_error(self, error: Exception, phase: str) -> None: + # Handle system overload condition. When the base AutoReconnect is + # raised and we are not an sdam pool, add to backoff and add the + # appropriate error label. + if not self.is_sdam and type(error) == AutoReconnect: + self._backoff += 1 + error._add_error_label("SystemOverloaded") + error._add_error_label("Retryable") + print(f"Setting backoff in {phase}:", self._backoff) # noqa: T201 + async def connect(self, handler: Optional[_MongoClientErrorHandler] = None) -> AsyncConnection: """Connect to Mongo and return a new AsyncConnection. @@ -1066,7 +1076,6 @@ async def connect(self, handler: Optional[_MongoClientErrorHandler] = None) -> A networking_interface = await _configured_protocol_interface(self.address, self.opts) # Catch KeyboardInterrupt, CancelledError, etc. and cleanup. except BaseException as error: - print("Got the TLS handshake error") # noqa: T201 async with self.lock: self.active_contexts.discard(tmp_context) if self.enabled_for_cmap: @@ -1085,10 +1094,10 @@ async def connect(self, handler: Optional[_MongoClientErrorHandler] = None) -> A reason=_verbose_connection_error_reason(ConnectionClosedReason.ERROR), error=ConnectionClosedReason.ERROR, ) + self._handle_connection_error(error, "handshake") if isinstance(error, (IOError, OSError, *SSLErrors)): details = _get_timeout_details(self.opts) _raise_connection_failure(self.address, error, timeout_details=details) - raise conn = AsyncConnection(networking_interface, self, self.address, conn_id, self.is_sdam) # type: ignore[arg-type] @@ -1109,14 +1118,7 @@ async def connect(self, handler: Optional[_MongoClientErrorHandler] = None) -> A except BaseException as e: async with self.lock: self.active_contexts.discard(conn.cancel_context) - # Handle system overload condition. When the base AutoReconnect is - # raised and we are not an sdam pool, add to backoff and add the - # appropriate error label. - if not self.is_sdam and type(e) == AutoReconnect: - self._backoff += 1 - e._add_error_label("SystemOverloaded") - e._add_error_label("Retryable") - print("Setting backoff:", self._backoff) # noqa: T201 + self._handle_connection_error(e, "hello") await conn.close_conn(ConnectionClosedReason.ERROR) raise diff --git a/pymongo/network_layer.py b/pymongo/network_layer.py index e15257c82b..582a085186 100644 --- a/pymongo/network_layer.py +++ b/pymongo/network_layer.py @@ -254,7 +254,7 @@ class PyMongoBaseProtocol(Protocol): def __init__(self, timeout: Optional[float] = None): self.transport: Transport = None # type: ignore[assignment] self._timeout = timeout - self._connection_made = asyncio.get_running_loop().create_future() + self._closing_error = asyncio.get_running_loop().create_future() self._closed = asyncio.get_running_loop().create_future() self._connection_lost = False @@ -271,13 +271,9 @@ def close(self, exc: Optional[Exception] = None) -> None: self._resolve_pending(exc) self._connection_lost = True - def connection_made(self, transport: BaseTransport) -> None: - super().connection_made(transport) - self._connection_made.set_result(None) - def connection_lost(self, exc: Optional[Exception] = None) -> None: - if exc is not None and not self._connection_made.done(): - self._connection_made.set_exception(exc) + if exc is not None and not self._closing_error.done(): + self._closing_error.set_exception(exc) self._resolve_pending(exc) if not self._closed.done(): self._closed.set_result(None) @@ -344,7 +340,7 @@ async def read(self, request_id: Optional[int], max_message_size: int) -> tuple[ message = await self._done_messages.popleft() else: if self.transport and self.transport.is_closing(): - raise OSError("connection is already closed") + return await self._closing_error read_waiter = asyncio.get_running_loop().create_future() self._pending_messages.append(read_waiter) try: diff --git a/pymongo/pool_shared.py b/pymongo/pool_shared.py index a4b4812335..0536dc3835 100644 --- a/pymongo/pool_shared.py +++ b/pymongo/pool_shared.py @@ -278,7 +278,6 @@ async def _configured_protocol_interface( server_hostname=host, ssl=ssl_context, ) - await protocol._connection_made except _CertificateError: # Raise _CertificateError directly like we do after match_hostname # below. diff --git a/pymongo/synchronous/pool.py b/pymongo/synchronous/pool.py index dbacc63b6a..2538b2b278 100644 --- a/pymongo/synchronous/pool.py +++ b/pymongo/synchronous/pool.py @@ -1025,6 +1025,16 @@ def remove_stale_sockets(self, reference_generation: int) -> None: self.requests -= 1 self.size_cond.notify() + def _handle_connection_error(self, error: Exception, phase: str) -> None: + # Handle system overload condition. When the base AutoReconnect is + # raised and we are not an sdam pool, add to backoff and add the + # appropriate error label. + if not self.is_sdam and type(error) == AutoReconnect: + self._backoff += 1 + error._add_error_label("SystemOverloaded") + error._add_error_label("Retryable") + print(f"Setting backoff in {phase}:", self._backoff) # noqa: T201 + def connect(self, handler: Optional[_MongoClientErrorHandler] = None) -> Connection: """Connect to Mongo and return a new Connection. @@ -1062,7 +1072,6 @@ def connect(self, handler: Optional[_MongoClientErrorHandler] = None) -> Connect networking_interface = _configured_socket_interface(self.address, self.opts) # Catch KeyboardInterrupt, CancelledError, etc. and cleanup. except BaseException as error: - print("Got the TLS handshake error") # noqa: T201 with self.lock: self.active_contexts.discard(tmp_context) if self.enabled_for_cmap: @@ -1081,10 +1090,10 @@ def connect(self, handler: Optional[_MongoClientErrorHandler] = None) -> Connect reason=_verbose_connection_error_reason(ConnectionClosedReason.ERROR), error=ConnectionClosedReason.ERROR, ) + self._handle_connection_error(error, "handshake") if isinstance(error, (IOError, OSError, *SSLErrors)): details = _get_timeout_details(self.opts) _raise_connection_failure(self.address, error, timeout_details=details) - raise conn = Connection(networking_interface, self, self.address, conn_id, self.is_sdam) # type: ignore[arg-type] @@ -1105,14 +1114,7 @@ def connect(self, handler: Optional[_MongoClientErrorHandler] = None) -> Connect except BaseException as e: with self.lock: self.active_contexts.discard(conn.cancel_context) - # Handle system overload condition. When the base AutoReconnect is - # raised and we are not an sdam pool, add to backoff and add the - # appropriate error label. - if not self.is_sdam and type(e) == AutoReconnect: - self._backoff += 1 - e._add_error_label("SystemOverloaded") - e._add_error_label("Retryable") - print("Setting backoff:", self._backoff) # noqa: T201 + self._handle_connection_error(e, "hello") conn.close_conn(ConnectionClosedReason.ERROR) raise From 679807e0e34e5434fd874a09638890d81307e58c Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 11 Sep 2025 05:48:38 -0500 Subject: [PATCH 49/53] update backoff criteria --- pymongo/asynchronous/pool.py | 2 +- pymongo/synchronous/pool.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pymongo/asynchronous/pool.py b/pymongo/asynchronous/pool.py index fd21add0d4..681d0456f6 100644 --- a/pymongo/asynchronous/pool.py +++ b/pymongo/asynchronous/pool.py @@ -1033,7 +1033,7 @@ def _handle_connection_error(self, error: Exception, phase: str) -> None: # Handle system overload condition. When the base AutoReconnect is # raised and we are not an sdam pool, add to backoff and add the # appropriate error label. - if not self.is_sdam and type(error) == AutoReconnect: + if not self.is_sdam and "[Errno 54] Connection reset by peer" in str(error): self._backoff += 1 error._add_error_label("SystemOverloaded") error._add_error_label("Retryable") diff --git a/pymongo/synchronous/pool.py b/pymongo/synchronous/pool.py index 2538b2b278..763a5c8cc0 100644 --- a/pymongo/synchronous/pool.py +++ b/pymongo/synchronous/pool.py @@ -1029,7 +1029,7 @@ def _handle_connection_error(self, error: Exception, phase: str) -> None: # Handle system overload condition. When the base AutoReconnect is # raised and we are not an sdam pool, add to backoff and add the # appropriate error label. - if not self.is_sdam and type(error) == AutoReconnect: + if not self.is_sdam and "[Errno 54] Connection reset by peer" in str(error): self._backoff += 1 error._add_error_label("SystemOverloaded") error._add_error_label("Retryable") From 7e9f19f376f821bae3246684d5551a653793208d Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 11 Sep 2025 05:49:29 -0500 Subject: [PATCH 50/53] update backoff criteria --- pymongo/asynchronous/pool.py | 2 +- pymongo/synchronous/pool.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pymongo/asynchronous/pool.py b/pymongo/asynchronous/pool.py index 681d0456f6..7d3df1f22e 100644 --- a/pymongo/asynchronous/pool.py +++ b/pymongo/asynchronous/pool.py @@ -1033,7 +1033,7 @@ def _handle_connection_error(self, error: Exception, phase: str) -> None: # Handle system overload condition. When the base AutoReconnect is # raised and we are not an sdam pool, add to backoff and add the # appropriate error label. - if not self.is_sdam and "[Errno 54] Connection reset by peer" in str(error): + if not self.is_sdam and "connection reset by peer" in str(error).lower(): self._backoff += 1 error._add_error_label("SystemOverloaded") error._add_error_label("Retryable") diff --git a/pymongo/synchronous/pool.py b/pymongo/synchronous/pool.py index 763a5c8cc0..7cee40bd5a 100644 --- a/pymongo/synchronous/pool.py +++ b/pymongo/synchronous/pool.py @@ -1029,7 +1029,7 @@ def _handle_connection_error(self, error: Exception, phase: str) -> None: # Handle system overload condition. When the base AutoReconnect is # raised and we are not an sdam pool, add to backoff and add the # appropriate error label. - if not self.is_sdam and "[Errno 54] Connection reset by peer" in str(error): + if not self.is_sdam and "connection reset by peer" in str(error).lower(): self._backoff += 1 error._add_error_label("SystemOverloaded") error._add_error_label("Retryable") From b0b58001c954b05d01e59a65cde4e4c3fa2a23ad Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 11 Sep 2025 06:42:21 -0500 Subject: [PATCH 51/53] update backoff criteria --- pymongo/asynchronous/pool.py | 6 +++++- pymongo/network_layer.py | 11 ++++------- pymongo/synchronous/pool.py | 6 +++++- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/pymongo/asynchronous/pool.py b/pymongo/asynchronous/pool.py index 7d3df1f22e..fbfcf0af3d 100644 --- a/pymongo/asynchronous/pool.py +++ b/pymongo/asynchronous/pool.py @@ -1033,7 +1033,11 @@ def _handle_connection_error(self, error: Exception, phase: str) -> None: # Handle system overload condition. When the base AutoReconnect is # raised and we are not an sdam pool, add to backoff and add the # appropriate error label. - if not self.is_sdam and "connection reset by peer" in str(error).lower(): + if ( + not self.is_sdam + and "connection reset by peer" in str(error).lower() + or ("connection closed" in str(error).lower() and self._backoff) + ): self._backoff += 1 error._add_error_label("SystemOverloaded") error._add_error_label("Retryable") diff --git a/pymongo/network_layer.py b/pymongo/network_layer.py index 582a085186..ee266d5df0 100644 --- a/pymongo/network_layer.py +++ b/pymongo/network_layer.py @@ -254,9 +254,9 @@ class PyMongoBaseProtocol(Protocol): def __init__(self, timeout: Optional[float] = None): self.transport: Transport = None # type: ignore[assignment] self._timeout = timeout - self._closing_error = asyncio.get_running_loop().create_future() self._closed = asyncio.get_running_loop().create_future() self._connection_lost = False + self._closing_exception = None def settimeout(self, timeout: float | None) -> None: self._timeout = timeout @@ -270,11 +270,11 @@ def close(self, exc: Optional[Exception] = None) -> None: self.transport.abort() self._resolve_pending(exc) self._connection_lost = True + self._closing_exception = exc def connection_lost(self, exc: Optional[Exception] = None) -> None: - if exc is not None and not self._closing_error.done(): - self._closing_error.set_exception(exc) self._resolve_pending(exc) + self._closing_exception = exc if not self._closed.done(): self._closed.set_result(None) @@ -325,7 +325,6 @@ def connection_made(self, transport: BaseTransport) -> None: """ self.transport = transport # type: ignore[assignment] self.transport.set_write_buffer_limits(MAX_MESSAGE_SIZE, MAX_MESSAGE_SIZE) - super().connection_made(self) async def read(self, request_id: Optional[int], max_message_size: int) -> tuple[bytes, int]: """Read a single MongoDB Wire Protocol message from this connection.""" @@ -339,8 +338,6 @@ async def read(self, request_id: Optional[int], max_message_size: int) -> tuple[ if self._done_messages: message = await self._done_messages.popleft() else: - if self.transport and self.transport.is_closing(): - return await self._closing_error read_waiter = asyncio.get_running_loop().create_future() self._pending_messages.append(read_waiter) try: @@ -478,6 +475,7 @@ def _resolve_pending(self, exc: Optional[Exception] = None) -> None: else: msg.set_exception(exc) self._done_messages.append(msg) + self._pending_messages.clear() class PyMongoKMSProtocol(PyMongoBaseProtocol): @@ -493,7 +491,6 @@ def connection_made(self, transport: BaseTransport) -> None: The transport argument is the transport representing the write side of the connection. """ self.transport = transport # type: ignore[assignment] - super().connection_made(self) def data_received(self, data: bytes) -> None: if self._connection_lost: diff --git a/pymongo/synchronous/pool.py b/pymongo/synchronous/pool.py index 7cee40bd5a..915e6ebc85 100644 --- a/pymongo/synchronous/pool.py +++ b/pymongo/synchronous/pool.py @@ -1029,7 +1029,11 @@ def _handle_connection_error(self, error: Exception, phase: str) -> None: # Handle system overload condition. When the base AutoReconnect is # raised and we are not an sdam pool, add to backoff and add the # appropriate error label. - if not self.is_sdam and "connection reset by peer" in str(error).lower(): + if ( + not self.is_sdam + and "connection reset by peer" in str(error).lower() + or ("connection closed" in str(error).lower() and self._backoff) + ): self._backoff += 1 error._add_error_label("SystemOverloaded") error._add_error_label("Retryable") From a033c58b3d7ff9f47fd34edb511ac184989c5e5a Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 11 Sep 2025 09:20:44 -0500 Subject: [PATCH 52/53] handle the already closed case --- pymongo/network_layer.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pymongo/network_layer.py b/pymongo/network_layer.py index ee266d5df0..d7e3d7785c 100644 --- a/pymongo/network_layer.py +++ b/pymongo/network_layer.py @@ -338,6 +338,8 @@ async def read(self, request_id: Optional[int], max_message_size: int) -> tuple[ if self._done_messages: message = await self._done_messages.popleft() else: + if self._closing_exception: + raise self._closing_exception read_waiter = asyncio.get_running_loop().create_future() self._pending_messages.append(read_waiter) try: From ded90b040df1c4eef69c1711ba567e0e4cec97be Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 11 Sep 2025 09:40:17 -0500 Subject: [PATCH 53/53] handle another edge case --- pymongo/network_layer.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pymongo/network_layer.py b/pymongo/network_layer.py index d7e3d7785c..54fe020563 100644 --- a/pymongo/network_layer.py +++ b/pymongo/network_layer.py @@ -338,8 +338,11 @@ async def read(self, request_id: Optional[int], max_message_size: int) -> tuple[ if self._done_messages: message = await self._done_messages.popleft() else: - if self._closing_exception: - raise self._closing_exception + if self._closed.done(): + if self._closing_exception: + raise self._closing_exception + else: + raise OSError("connection closed") read_waiter = asyncio.get_running_loop().create_future() self._pending_messages.append(read_waiter) try: