From c74afb8544bce38dcd33edd5fd331f5a5bfcba3c Mon Sep 17 00:00:00 2001 From: bailey Date: Fri, 21 Nov 2025 10:59:35 -0700 Subject: [PATCH 01/10] add token bucket --- src/token_bucket.ts | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 src/token_bucket.ts diff --git a/src/token_bucket.ts b/src/token_bucket.ts new file mode 100644 index 0000000000..fddacdb657 --- /dev/null +++ b/src/token_bucket.ts @@ -0,0 +1,20 @@ +export class TokenBucket { + private budget: number; + constructor(allowance: number) { + this.budget = allowance; + } + deposit(tokens: number) { + this.budget += tokens; + } + + consume(tokens: number): boolean { + if (tokens > this.budget) return false; + + this.budget -= tokens; + return true; + } +} + +export const TOKEN_REFRESH_RATE = 0.1; +export const INITIAL_SIZE = 1000; +export const RETRY_COST = 1; From 525f5be01951c89df4ed0db9ba5989bc2e176fbe Mon Sep 17 00:00:00 2001 From: bailey Date: Tue, 25 Nov 2025 10:57:21 -0700 Subject: [PATCH 02/10] add new error labels --- src/error.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/error.ts b/src/error.ts index 9822361e72..ff6f0932af 100644 --- a/src/error.ts +++ b/src/error.ts @@ -99,7 +99,9 @@ export const MongoErrorLabel = Object.freeze({ ResetPool: 'ResetPool', PoolRequestedRetry: 'PoolRequestedRetry', InterruptInUseConnections: 'InterruptInUseConnections', - NoWritesPerformed: 'NoWritesPerformed' + NoWritesPerformed: 'NoWritesPerformed', + SystemOverloadError: 'SystemOverloadError', + RetryableError: 'RetryableError' } as const); /** @public */ From 399a95d25d8d57d00c1f28888df444b0d89a089f Mon Sep 17 00:00:00 2001 From: bailey Date: Tue, 25 Nov 2025 12:10:24 -0700 Subject: [PATCH 03/10] use token bucket and apply exponential backoff in retry loop --- src/operations/execute_operation.ts | 160 ++++++++++++++++++++-------- src/sdam/topology.ts | 8 +- 2 files changed, 116 insertions(+), 52 deletions(-) diff --git a/src/operations/execute_operation.ts b/src/operations/execute_operation.ts index d833623d14..e93f5f3c7e 100644 --- a/src/operations/execute_operation.ts +++ b/src/operations/execute_operation.ts @@ -1,3 +1,5 @@ +import { setTimeout } from 'node:timers/promises'; + import { MIN_SUPPORTED_SNAPSHOT_READS_WIRE_VERSION } from '../cmap/wire_protocol/constants'; import { isRetryableReadError, @@ -10,6 +12,7 @@ import { MongoInvalidArgumentError, MongoNetworkError, MongoNotConnectedError, + MongoOperationTimeoutError, MongoRuntimeError, MongoServerError, MongoTransactionError, @@ -26,6 +29,7 @@ import { import type { Topology } from '../sdam/topology'; import type { ClientSession } from '../sessions'; import { TimeoutContext } from '../timeout'; +import { RETRY_COST, TOKEN_REFRESH_RATE } from '../token_bucket'; import { abortable, maxWireVersion, supportsRetryableWrites } from '../utils'; import { AggregateOperation } from './aggregate'; import { AbstractOperation, Aspect } from './operation'; @@ -232,71 +236,138 @@ async function tryOperation maxSystemOverloadRetryAttempts) { + throw previousOperationError; + } + + const delayMS = + Math.random() * + Math.min( + 10_000, // MAX_BACKOFF, + 100 * 2 ** systemOverloadRetryAttempt + ); + + // if the delay would exhaust the CSOT timeout, short-circuit. + if (timeoutContext.csotEnabled() && delayMS > timeoutContext.remainingTimeMS) { + // TODO: is this the right error to throw? + throw new MongoOperationTimeoutError( + `MongoDB SystemOverload exponential backoff would exceed timeoutMS deadline: remaining CSOT deadline=${timeoutContext.remainingTimeMS}, backoff delayMS=${delayMS}`, + { + cause: previousOperationError + } + ); + } + + await setTimeout(delayMS); + + if (!topology.tokenBucket.consume(RETRY_COST)) { + throw previousOperationError; + } + + server = await topology.selectServer(selector, { + session, + operationName: operation.commandName, + previousServer, + signal: operation.options.signal + }); + } else { + // we have no more retry attempts, throw. + if (nonOverloadRetryAttempt >= maxNonOverloadRetryAttempts) { + throw previousOperationError; + } + + if (hasWriteAspect && previousOperationError.code === MMAPv1_RETRY_WRITES_ERROR_CODE) { + throw new MongoServerError({ + message: MMAPv1_RETRY_WRITES_ERROR_MESSAGE, + errmsg: MMAPv1_RETRY_WRITES_ERROR_MESSAGE, + originalError: previousOperationError + }); + } + + if ( + (operation.hasAspect(Aspect.COMMAND_BATCHING) && !operation.canRetryWrite) || + (hasWriteAspect && !isRetryableWriteError(previousOperationError)) || + (hasReadAspect && !isRetryableReadError(previousOperationError)) + ) { + throw previousOperationError; + } + + if ( + previousOperationError instanceof MongoNetworkError && + operation.hasAspect(Aspect.CURSOR_CREATING) && + session != null && + session.isPinned && + !session.inTransaction() + ) { + session.unpin({ force: true, forceClear: true }); + } + + server = await topology.selectServer(selector, { + session, + operationName: operation.commandName, + previousServer, + signal: operation.options.signal }); - } - - if (operation.hasAspect(Aspect.COMMAND_BATCHING) && !operation.canRetryWrite) { - throw previousOperationError; - } - - if (hasWriteAspect && !isRetryableWriteError(previousOperationError)) - throw previousOperationError; - - if (hasReadAspect && !isRetryableReadError(previousOperationError)) { - throw previousOperationError; - } - - if ( - previousOperationError instanceof MongoNetworkError && - operation.hasAspect(Aspect.CURSOR_CREATING) && - session != null && - session.isPinned && - !session.inTransaction() - ) { - session.unpin({ force: true, forceClear: true }); - } - - server = await topology.selectServer(selector, { - session, - operationName: operation.commandName, - previousServer, - signal: operation.options.signal - }); - if (hasWriteAspect && !supportsRetryableWrites(server)) { - throw new MongoUnexpectedServerResponseError( - 'Selected server does not support retryable writes' - ); + if (hasWriteAspect && !supportsRetryableWrites(server)) { + throw new MongoUnexpectedServerResponseError( + 'Selected server does not support retryable writes' + ); + } } } operation.server = server; try { - // If tries > 0 and we are command batching we need to reset the batch. - if (tries > 0 && operation.hasAspect(Aspect.COMMAND_BATCHING)) { + // If attempt > 0 and we are command batching we need to reset the batch. + if (nonOverloadRetryAttempt > 0 && operation.hasAspect(Aspect.COMMAND_BATCHING)) { operation.resetBatch(); } try { const result = await server.command(operation, timeoutContext); + const isRetry = nonOverloadRetryAttempt > 0 || systemOverloadRetryAttempt > 0; + topology.tokenBucket.deposit( + isRetry + ? // on successful retry, deposit the retry cost + the refresh rate. + TOKEN_REFRESH_RATE + RETRY_COST + : // otherwise, just deposit the refresh rate. + TOKEN_REFRESH_RATE + ); return operation.handleOk(result); } catch (error) { return operation.handleError(error); } } catch (operationError) { if (!(operationError instanceof MongoError)) throw operationError; + + if (!operationError.hasErrorLabel(MongoErrorLabel.SystemOverloadError)) { + // if an operation fails with an error that does not contain the SystemOverloadError, deposit 1 token. + topology.tokenBucket.deposit(RETRY_COST); + } + if ( previousOperationError != null && operationError.hasErrorLabel(MongoErrorLabel.NoWritesPerformed) @@ -310,9 +381,4 @@ async function tryOperation { - /** @internal */ s: TopologyPrivate; - /** @internal */ waitQueue: List; - /** @internal */ hello?: Document; - /** @internal */ _type?: string; + tokenBucket = new TokenBucket(1000); + client!: MongoClient; - /** @internal */ private connectionLock?: Promise; /** @event */ From b28f772385fa348034f1893c9a908287fbbc72fe Mon Sep 17 00:00:00 2001 From: bailey Date: Tue, 25 Nov 2025 14:20:16 -0700 Subject: [PATCH 04/10] retryability cleanup changes --- src/operations/execute_operation.ts | 32 +++++++++++++++++++++-------- src/utils.ts | 10 +++++++++ 2 files changed, 33 insertions(+), 9 deletions(-) diff --git a/src/operations/execute_operation.ts b/src/operations/execute_operation.ts index e93f5f3c7e..9d0e9e90fe 100644 --- a/src/operations/execute_operation.ts +++ b/src/operations/execute_operation.ts @@ -30,7 +30,12 @@ import type { Topology } from '../sdam/topology'; import type { ClientSession } from '../sessions'; import { TimeoutContext } from '../timeout'; import { RETRY_COST, TOKEN_REFRESH_RATE } from '../token_bucket'; -import { abortable, maxWireVersion, supportsRetryableWrites } from '../utils'; +import { + abortable, + exponentialBackoffDelayProvider, + maxWireVersion, + supportsRetryableWrites +} from '../utils'; import { AggregateOperation } from './aggregate'; import { AbstractOperation, Aspect } from './operation'; @@ -245,14 +250,28 @@ async function tryOperation timeoutContext.remainingTimeMS) { diff --git a/src/utils.ts b/src/utils.ts index 49aa15ccea..82df23c351 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1428,3 +1428,13 @@ export async function abortable( abortListener?.[kDispose](); } } + +export function* exponentialBackoffDelayProvider( + maxBackoff: number, + baseBackoff: number, + backoffIncreaseRate: number +): Generator { + for (let i = 0; ; i++) { + yield Math.random() * Math.min(maxBackoff, baseBackoff * backoffIncreaseRate ** i); + } +} From 59ed38fb182d569134d63ca8af2d1635cb4e260f Mon Sep 17 00:00:00 2001 From: bailey Date: Tue, 25 Nov 2025 16:32:46 -0700 Subject: [PATCH 05/10] initial POC --- src/operations/execute_operation.ts | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/operations/execute_operation.ts b/src/operations/execute_operation.ts index 9d0e9e90fe..0c7dc7d8ce 100644 --- a/src/operations/execute_operation.ts +++ b/src/operations/execute_operation.ts @@ -59,7 +59,7 @@ type ResultTypeFromOperation = ReturnType< * The expectation is that this function: * - Connects the MongoClient if it has not already been connected, see {@link autoConnect} * - Creates a session if none is provided and cleans up the session it creates - * - Tries an operation and retries under certain conditions, see {@link tryOperation} + * - Tries an operation and retries under certain conditions, see {@link executeOperationWithRetries} * * @typeParam T - The operation's type * @typeParam TResult - The type of the operation's result, calculated from T @@ -129,7 +129,7 @@ export async function executeOperation< }); try { - return await tryOperation(operation, { + return await executeOperationWithRetries(operation, { topology, timeoutContext, session, @@ -193,7 +193,10 @@ type RetryOptions = { * * @param operation - The operation to execute * */ -async function tryOperation>( +async function executeOperationWithRetries< + T extends AbstractOperation, + TResult = ResultTypeFromOperation +>( operation: T, { topology, timeoutContext, session, readPreference }: RetryOptions ): Promise { @@ -246,7 +249,7 @@ async function tryOperation= maxNonOverloadRetryAttempts) { + if (nonOverloadRetryAttempt > maxNonOverloadRetryAttempts) { throw previousOperationError; } From 6aa508625efe2cb58ffc24b50f0cdf59655a33c2 Mon Sep 17 00:00:00 2001 From: bailey Date: Mon, 1 Dec 2025 13:49:39 -0700 Subject: [PATCH 06/10] fix tests and rename SystemOverloadError -> SystemOverloadedError --- src/error.ts | 2 +- src/operations/execute_operation.ts | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/error.ts b/src/error.ts index ff6f0932af..f6dc68b9f4 100644 --- a/src/error.ts +++ b/src/error.ts @@ -100,7 +100,7 @@ export const MongoErrorLabel = Object.freeze({ PoolRequestedRetry: 'PoolRequestedRetry', InterruptInUseConnections: 'InterruptInUseConnections', NoWritesPerformed: 'NoWritesPerformed', - SystemOverloadError: 'SystemOverloadError', + SystemOverloadedError: 'SystemOverloadedError', RetryableError: 'RetryableError' } as const); diff --git a/src/operations/execute_operation.ts b/src/operations/execute_operation.ts index 0c7dc7d8ce..c8a24a8299 100644 --- a/src/operations/execute_operation.ts +++ b/src/operations/execute_operation.ts @@ -261,7 +261,7 @@ async function executeOperationWithRetries< while (true) { if (previousOperationError) { - if (previousOperationError.hasErrorLabel(MongoErrorLabel.SystemOverloadError)) { + if (previousOperationError.hasErrorLabel(MongoErrorLabel.SystemOverloadedError)) { systemOverloadRetryAttempt += 1; if ( @@ -279,7 +279,7 @@ async function executeOperationWithRetries< } // if we have exhausted overload retry attempts, throw. - if (systemOverloadRetryAttempt > maxSystemOverloadRetryAttempts) { + if (systemOverloadRetryAttempt >= maxSystemOverloadRetryAttempts) { throw previousOperationError; } @@ -311,7 +311,7 @@ async function executeOperationWithRetries< } else { nonOverloadRetryAttempt++; // we have no more retry attempts, throw. - if (nonOverloadRetryAttempt > maxNonOverloadRetryAttempts) { + if (nonOverloadRetryAttempt >= maxNonOverloadRetryAttempts) { throw previousOperationError; } @@ -381,7 +381,7 @@ async function executeOperationWithRetries< } catch (operationError) { if (!(operationError instanceof MongoError)) throw operationError; - if (!operationError.hasErrorLabel(MongoErrorLabel.SystemOverloadError)) { + if (!operationError.hasErrorLabel(MongoErrorLabel.SystemOverloadedError)) { // if an operation fails with an error that does not contain the SystemOverloadError, deposit 1 token. topology.tokenBucket.deposit(RETRY_COST); } From 5ffe949d1ac37d07cfbaa0362369df075ed4e45b Mon Sep 17 00:00:00 2001 From: bailey Date: Mon, 1 Dec 2025 13:52:15 -0700 Subject: [PATCH 07/10] Add tests for retry loop and backoff. --- src/operations/execute_operation.ts | 20 +- sync.sh | 3 + .../client-backpressure.spec.test.ts | 16 + test/spec/client-backpressure/README.md | 15 + .../backpressure-retry-loop.json | 3403 +++++++++++++++++ .../backpressure-retry-loop.yml | 1860 +++++++++ .../backpressure-retry-loop.yml.template | 128 + .../backpressure-retry-max-attempts.json | 1262 ++++++ .../backpressure-retry-max-attempts.yml | 753 ++++ ...ckpressure-retry-max-attempts.yml.template | 72 + 10 files changed, 7524 insertions(+), 8 deletions(-) create mode 100644 sync.sh create mode 100644 test/integration/client-backpressure/client-backpressure.spec.test.ts create mode 100644 test/spec/client-backpressure/README.md create mode 100644 test/spec/client-backpressure/backpressure-retry-loop.json create mode 100644 test/spec/client-backpressure/backpressure-retry-loop.yml create mode 100644 test/spec/client-backpressure/backpressure-retry-loop.yml.template create mode 100644 test/spec/client-backpressure/backpressure-retry-max-attempts.json create mode 100644 test/spec/client-backpressure/backpressure-retry-max-attempts.yml create mode 100644 test/spec/client-backpressure/backpressure-retry-max-attempts.yml.template diff --git a/src/operations/execute_operation.ts b/src/operations/execute_operation.ts index c8a24a8299..6094c86b75 100644 --- a/src/operations/execute_operation.ts +++ b/src/operations/execute_operation.ts @@ -38,6 +38,7 @@ import { } from '../utils'; import { AggregateOperation } from './aggregate'; import { AbstractOperation, Aspect } from './operation'; +import { RunCommandOperation } from './run_command'; const MMAPv1_RETRY_WRITES_ERROR_CODE = MONGODB_ERROR_CODES.IllegalOperation; const MMAPv1_RETRY_WRITES_ERROR_MESSAGE = @@ -264,16 +265,16 @@ async function executeOperationWithRetries< if (previousOperationError.hasErrorLabel(MongoErrorLabel.SystemOverloadedError)) { systemOverloadRetryAttempt += 1; + // if retryable writes or reads are not configured, throw. + const isOperationConfiguredForRetry = + (hasReadAspect && topology.s.options.retryReads) || + (hasWriteAspect && topology.s.options.retryWrites); + const isRunCommand = operation instanceof RunCommandOperation; + if ( // if the SystemOverloadError is not retryable, throw. !previousOperationError.hasErrorLabel(MongoErrorLabel.RetryableError) || - !( - // if retryable writes or reads are not configured, throw. - ( - (hasReadAspect && topology.s.options.retryReads) || - (hasWriteAspect && topology.s.options.retryWrites) - ) - ) + !(isOperationConfiguredForRetry || isRunCommand) ) { throw previousOperationError; } @@ -360,7 +361,10 @@ async function executeOperationWithRetries< try { // If attempt > 0 and we are command batching we need to reset the batch. - if (nonOverloadRetryAttempt > 0 && operation.hasAspect(Aspect.COMMAND_BATCHING)) { + if ( + (nonOverloadRetryAttempt > 0 || systemOverloadRetryAttempt > 0) && + operation.hasAspect(Aspect.COMMAND_BATCHING) + ) { operation.resetBatch(); } diff --git a/sync.sh b/sync.sh new file mode 100644 index 0000000000..aab954ba4e --- /dev/null +++ b/sync.sh @@ -0,0 +1,3 @@ + + +cp ~/dev/specifications/source/client-backpressure/tests/* ~/dev/node-mongodb-native/test/spec/client-backpressure \ No newline at end of file diff --git a/test/integration/client-backpressure/client-backpressure.spec.test.ts b/test/integration/client-backpressure/client-backpressure.spec.test.ts new file mode 100644 index 0000000000..4175929252 --- /dev/null +++ b/test/integration/client-backpressure/client-backpressure.spec.test.ts @@ -0,0 +1,16 @@ +import { loadSpecTests } from '../../spec'; +import { runUnifiedSuite } from '../../tools/unified-spec-runner/runner'; +import { type Test } from '../../tools/unified-spec-runner/schema'; + +const skippedTests = { + 'collection.dropIndexes retries at most maxAttempts times (maxAttempts=5)': + 'TODO(NODE-6517): dropIndexes squashes all errors other than ns not found' +}; + +function shouldSkip({ description }: Test) { + return skippedTests[description] ?? false; +} + +describe('Client Backpressure (spec)', function () { + runUnifiedSuite(loadSpecTests('client-backpressure'), shouldSkip); +}); diff --git a/test/spec/client-backpressure/README.md b/test/spec/client-backpressure/README.md new file mode 100644 index 0000000000..7a70e4ad76 --- /dev/null +++ b/test/spec/client-backpressure/README.md @@ -0,0 +1,15 @@ +# Client Backpressure Tests + +______________________________________________________________________ + +## Introduction + +The YAML and JSON files in this directory are platform-independent tests meant to exercise a driver's implementation of +retryable reads. These tests utilize the [Unified Test Format](../../unified-test-format/unified-test-format.md). + +Several prose tests, which are not easily expressed in YAML, are also presented in this file. Those tests will need to +be manually implemented by each driver. + +## Changelog + +- 2025-XX-XX: Initial version. diff --git a/test/spec/client-backpressure/backpressure-retry-loop.json b/test/spec/client-backpressure/backpressure-retry-loop.json new file mode 100644 index 0000000000..21fc802344 --- /dev/null +++ b/test/spec/client-backpressure/backpressure-retry-loop.json @@ -0,0 +1,3403 @@ +{ + "description": "tests that operations respect overload backoff retry loop", + "schemaVersion": "1.0", + "runOnRequirements": [ + { + "minServerVersion": "4.4", + "topologies": [ + "replicaset", + "sharded", + "load-balanced" + ] + } + ], + "createEntities": [ + { + "client": { + "id": "client", + "useMultipleMongoses": false, + "observeEvents": [ + "commandStartedEvent", + "commandSucceededEvent", + "commandFailedEvent" + ] + } + }, + { + "client": { + "id": "failPointClient", + "useMultipleMongoses": false + } + }, + { + "database": { + "id": "utilDb", + "client": "failPointClient", + "databaseName": "retryable-writes-tests" + } + }, + { + "collection": { + "id": "utilCollection", + "database": "utilDb", + "collectionName": "coll" + } + }, + { + "database": { + "id": "database", + "client": "client", + "databaseName": "retryable-writes-tests" + } + }, + { + "collection": { + "id": "collection", + "database": "database", + "collectionName": "coll" + } + } + ], + "initialData": [ + { + "collectionName": "coll", + "databaseName": "retryable-writes-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + } + ] + } + ], + "tests": [ + { + "description": "client.listDatabases retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "listDatabases" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "client", + "name": "listDatabases", + "arguments": { + "filter": {} + }, + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandSucceededEvent": { + "commandName": "listDatabases" + } + } + ] + } + ] + }, + { + "description": "client.listDatabaseNames retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "listDatabases" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "client", + "name": "listDatabaseNames", + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandSucceededEvent": { + "commandName": "listDatabases" + } + } + ] + } + ] + }, + { + "description": "client.createChangeStream retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "client", + "name": "createChangeStream", + "arguments": { + "pipeline": [] + }, + "saveResultAsEntity": "changeStream", + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandSucceededEvent": { + "commandName": "aggregate" + } + } + ] + } + ] + }, + { + "description": "client.clientBulkWrite retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "bulkWrite" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "client", + "name": "clientBulkWrite", + "arguments": { + "models": [ + { + "insertOne": { + "namespace": "retryable-writes-tests.coll", + "document": { + "_id": 8, + "x": 88 + } + } + } + ] + }, + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandFailedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandStartedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandFailedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandStartedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandFailedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandStartedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandSucceededEvent": { + "commandName": "bulkWrite" + } + } + ] + } + ] + }, + { + "description": "database.aggregate retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "database", + "name": "aggregate", + "arguments": { + "pipeline": [ + { + "$listLocalSessions": {} + }, + { + "$limit": 1 + } + ] + }, + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandSucceededEvent": { + "commandName": "aggregate" + } + } + ] + } + ] + }, + { + "description": "database.listCollections retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "listCollections" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "database", + "name": "listCollections", + "arguments": { + "filter": {} + }, + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandSucceededEvent": { + "commandName": "listCollections" + } + } + ] + } + ] + }, + { + "description": "database.listCollectionNames retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "listCollections" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "database", + "name": "listCollectionNames", + "arguments": { + "filter": {} + }, + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandSucceededEvent": { + "commandName": "listCollections" + } + } + ] + } + ] + }, + { + "description": "database.runCommand retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "ping" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "database", + "name": "runCommand", + "arguments": { + "command": { + "ping": 1 + }, + "commandName": "ping" + }, + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "ping" + } + }, + { + "commandFailedEvent": { + "commandName": "ping" + } + }, + { + "commandStartedEvent": { + "commandName": "ping" + } + }, + { + "commandFailedEvent": { + "commandName": "ping" + } + }, + { + "commandStartedEvent": { + "commandName": "ping" + } + }, + { + "commandFailedEvent": { + "commandName": "ping" + } + }, + { + "commandStartedEvent": { + "commandName": "ping" + } + }, + { + "commandSucceededEvent": { + "commandName": "ping" + } + } + ] + } + ] + }, + { + "description": "database.createChangeStream retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "database", + "name": "createChangeStream", + "arguments": { + "pipeline": [] + }, + "saveResultAsEntity": "changeStream", + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandSucceededEvent": { + "commandName": "aggregate" + } + } + ] + } + ] + }, + { + "description": "collection.aggregate retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "aggregate", + "arguments": { + "pipeline": [] + }, + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandSucceededEvent": { + "commandName": "aggregate" + } + } + ] + } + ] + }, + { + "description": "collection.countDocuments retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "countDocuments", + "arguments": { + "filter": {} + }, + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandSucceededEvent": { + "commandName": "aggregate" + } + } + ] + } + ] + }, + { + "description": "collection.estimatedDocumentCount retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "count" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "estimatedDocumentCount", + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "count" + } + }, + { + "commandFailedEvent": { + "commandName": "count" + } + }, + { + "commandStartedEvent": { + "commandName": "count" + } + }, + { + "commandFailedEvent": { + "commandName": "count" + } + }, + { + "commandStartedEvent": { + "commandName": "count" + } + }, + { + "commandFailedEvent": { + "commandName": "count" + } + }, + { + "commandStartedEvent": { + "commandName": "count" + } + }, + { + "commandSucceededEvent": { + "commandName": "count" + } + } + ] + } + ] + }, + { + "description": "collection.distinct retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "distinct" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "distinct", + "arguments": { + "fieldName": "x", + "filter": {} + }, + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "distinct" + } + }, + { + "commandFailedEvent": { + "commandName": "distinct" + } + }, + { + "commandStartedEvent": { + "commandName": "distinct" + } + }, + { + "commandFailedEvent": { + "commandName": "distinct" + } + }, + { + "commandStartedEvent": { + "commandName": "distinct" + } + }, + { + "commandFailedEvent": { + "commandName": "distinct" + } + }, + { + "commandStartedEvent": { + "commandName": "distinct" + } + }, + { + "commandSucceededEvent": { + "commandName": "distinct" + } + } + ] + } + ] + }, + { + "description": "collection.find retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "find" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "find", + "arguments": { + "filter": {} + }, + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandSucceededEvent": { + "commandName": "find" + } + } + ] + } + ] + }, + { + "description": "collection.findOne retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "find" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "findOne", + "arguments": { + "filter": {} + }, + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandSucceededEvent": { + "commandName": "find" + } + } + ] + } + ] + }, + { + "description": "collection.listIndexes retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "listIndexes" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "listIndexes", + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandSucceededEvent": { + "commandName": "listIndexes" + } + } + ] + } + ] + }, + { + "description": "collection.listIndexNames retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "listIndexes" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "listIndexNames", + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandSucceededEvent": { + "commandName": "listIndexes" + } + } + ] + } + ] + }, + { + "description": "collection.createChangeStream retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "createChangeStream", + "arguments": { + "pipeline": [] + }, + "saveResultAsEntity": "changeStream", + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandSucceededEvent": { + "commandName": "aggregate" + } + } + ] + } + ] + }, + { + "description": "collection.insertOne retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "insert" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "insertOne", + "arguments": { + "document": { + "_id": 2, + "x": 22 + } + }, + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandSucceededEvent": { + "commandName": "insert" + } + } + ] + } + ] + }, + { + "description": "collection.insertMany retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "insert" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "insertMany", + "arguments": { + "documents": [ + { + "_id": 2, + "x": 22 + } + ] + }, + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandSucceededEvent": { + "commandName": "insert" + } + } + ] + } + ] + }, + { + "description": "collection.deleteOne retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "delete" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "deleteOne", + "arguments": { + "filter": {} + }, + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandSucceededEvent": { + "commandName": "delete" + } + } + ] + } + ] + }, + { + "description": "collection.deleteMany retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "delete" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "deleteMany", + "arguments": { + "filter": {} + }, + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandSucceededEvent": { + "commandName": "delete" + } + } + ] + } + ] + }, + { + "description": "collection.replaceOne retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "update" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "replaceOne", + "arguments": { + "filter": {}, + "replacement": { + "x": 22 + } + }, + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandSucceededEvent": { + "commandName": "update" + } + } + ] + } + ] + }, + { + "description": "collection.updateOne retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "update" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "updateOne", + "arguments": { + "filter": {}, + "update": { + "$set": { + "x": 22 + } + } + }, + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandSucceededEvent": { + "commandName": "update" + } + } + ] + } + ] + }, + { + "description": "collection.updateMany retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "update" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "updateMany", + "arguments": { + "filter": {}, + "update": { + "$set": { + "x": 22 + } + } + }, + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandSucceededEvent": { + "commandName": "update" + } + } + ] + } + ] + }, + { + "description": "collection.findOneAndDelete retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "findAndModify" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "findOneAndDelete", + "arguments": { + "filter": {} + }, + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandSucceededEvent": { + "commandName": "findAndModify" + } + } + ] + } + ] + }, + { + "description": "collection.findOneAndReplace retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "findAndModify" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "findOneAndReplace", + "arguments": { + "filter": {}, + "replacement": { + "x": 22 + } + }, + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandSucceededEvent": { + "commandName": "findAndModify" + } + } + ] + } + ] + }, + { + "description": "collection.findOneAndUpdate retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "findAndModify" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "findOneAndUpdate", + "arguments": { + "filter": {}, + "update": { + "$set": { + "x": 22 + } + } + }, + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandSucceededEvent": { + "commandName": "findAndModify" + } + } + ] + } + ] + }, + { + "description": "collection.bulkWrite retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "insert" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "bulkWrite", + "arguments": { + "requests": [ + { + "insertOne": { + "document": { + "_id": 2, + "x": 22 + } + } + } + ] + }, + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandSucceededEvent": { + "commandName": "insert" + } + } + ] + } + ] + }, + { + "description": "collection.createIndex retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "createIndexes" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "createIndex", + "arguments": { + "keys": { + "x": 11 + }, + "name": "x_11" + }, + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandSucceededEvent": { + "commandName": "createIndexes" + } + } + ] + } + ] + }, + { + "description": "collection.dropIndex retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "object": "utilCollection", + "name": "createIndex", + "arguments": { + "keys": { + "x": 11 + }, + "name": "x_11" + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "dropIndexes" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "dropIndex", + "arguments": { + "name": "x_11" + }, + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandSucceededEvent": { + "commandName": "dropIndexes" + } + } + ] + } + ] + }, + { + "description": "collection.dropIndexes retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "dropIndexes" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "dropIndexes", + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandSucceededEvent": { + "commandName": "dropIndexes" + } + } + ] + } + ] + } + ] +} diff --git a/test/spec/client-backpressure/backpressure-retry-loop.yml b/test/spec/client-backpressure/backpressure-retry-loop.yml new file mode 100644 index 0000000000..ec2fa25b8f --- /dev/null +++ b/test/spec/client-backpressure/backpressure-retry-loop.yml @@ -0,0 +1,1860 @@ +# Tests in this file are generated from backpressure-retry-loop.yml.template. + +description: tests that operations respect overload backoff retry loop + +schemaVersion: '1.0' + +runOnRequirements: + - + minServerVersion: '4.4' # failCommand + topologies: [replicaset, sharded, load-balanced] + +createEntities: + - + client: + id: &client client + useMultipleMongoses: false + observeEvents: [ 'commandStartedEvent', 'commandSucceededEvent', 'commandFailedEvent' ] + + - + client: + id: &failPointClient failPointClient + useMultipleMongoses: false + + - + database: + id: &utilDb utilDb + client: *failPointClient + databaseName: &database_name retryable-writes-tests + + - + collection: + id: &utilCollection utilCollection + database: *utilDb + collectionName: &collection_name coll + + - + database: + id: &database database + client: *client + databaseName: &database_name retryable-writes-tests + - + collection: + id: &collection collection + database: *database + collectionName: &collection_name coll + +initialData: + - + collectionName: *collection_name + databaseName: *database_name + documents: + - { _id: 1, x: 11 } + - { _id: 2, x: 22 } + +tests: + + - + description: 'client.listDatabases retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [listDatabases] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *client + name: listDatabases + arguments: + filter: {} + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandSucceededEvent: + commandName: listDatabases + + - + description: 'client.listDatabaseNames retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [listDatabases] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *client + name: listDatabaseNames + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandSucceededEvent: + commandName: listDatabases + + - + description: 'client.createChangeStream retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [aggregate] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *client + name: createChangeStream + arguments: + pipeline: [] + saveResultAsEntity: changeStream + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandSucceededEvent: + commandName: aggregate + + - + description: 'client.clientBulkWrite retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [bulkWrite] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *client + name: clientBulkWrite + arguments: + models: + - insertOne: + namespace: retryable-writes-tests.coll + document: { _id: 8, x: 88 } + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: bulkWrite + - commandFailedEvent: + commandName: bulkWrite + - commandStartedEvent: + commandName: bulkWrite + - commandFailedEvent: + commandName: bulkWrite + - commandStartedEvent: + commandName: bulkWrite + - commandFailedEvent: + commandName: bulkWrite + - commandStartedEvent: + commandName: bulkWrite + - commandSucceededEvent: + commandName: bulkWrite + + - + description: 'database.aggregate retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [aggregate] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *database + name: aggregate + arguments: + pipeline: [ { $listLocalSessions: {} }, { $limit: 1 } ] + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandSucceededEvent: + commandName: aggregate + + - + description: 'database.listCollections retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [listCollections] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *database + name: listCollections + arguments: + filter: {} + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandSucceededEvent: + commandName: listCollections + + - + description: 'database.listCollectionNames retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [listCollections] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *database + name: listCollectionNames + arguments: + filter: {} + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandSucceededEvent: + commandName: listCollections + + - + description: 'database.runCommand retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [ping] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *database + name: runCommand + arguments: + command: { ping: 1 } + commandName: ping + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: ping + - commandFailedEvent: + commandName: ping + - commandStartedEvent: + commandName: ping + - commandFailedEvent: + commandName: ping + - commandStartedEvent: + commandName: ping + - commandFailedEvent: + commandName: ping + - commandStartedEvent: + commandName: ping + - commandSucceededEvent: + commandName: ping + + - + description: 'database.createChangeStream retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [aggregate] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *database + name: createChangeStream + arguments: + pipeline: [] + saveResultAsEntity: changeStream + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandSucceededEvent: + commandName: aggregate + + - + description: 'collection.aggregate retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [aggregate] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: aggregate + arguments: + pipeline: [] + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandSucceededEvent: + commandName: aggregate + + - + description: 'collection.countDocuments retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [aggregate] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: countDocuments + arguments: + filter: {} + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandSucceededEvent: + commandName: aggregate + + - + description: 'collection.estimatedDocumentCount retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [count] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: estimatedDocumentCount + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: count + - commandFailedEvent: + commandName: count + - commandStartedEvent: + commandName: count + - commandFailedEvent: + commandName: count + - commandStartedEvent: + commandName: count + - commandFailedEvent: + commandName: count + - commandStartedEvent: + commandName: count + - commandSucceededEvent: + commandName: count + + - + description: 'collection.distinct retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [distinct] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: distinct + arguments: + fieldName: x + filter: {} + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: distinct + - commandFailedEvent: + commandName: distinct + - commandStartedEvent: + commandName: distinct + - commandFailedEvent: + commandName: distinct + - commandStartedEvent: + commandName: distinct + - commandFailedEvent: + commandName: distinct + - commandStartedEvent: + commandName: distinct + - commandSucceededEvent: + commandName: distinct + + - + description: 'collection.find retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [find] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: find + arguments: + filter: {} + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandSucceededEvent: + commandName: find + + - + description: 'collection.findOne retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [find] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: findOne + arguments: + filter: {} + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandSucceededEvent: + commandName: find + + - + description: 'collection.listIndexes retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [listIndexes] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: listIndexes + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandSucceededEvent: + commandName: listIndexes + + - + description: 'collection.listIndexNames retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [listIndexes] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: listIndexNames + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandSucceededEvent: + commandName: listIndexes + + - + description: 'collection.createChangeStream retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [aggregate] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: createChangeStream + arguments: + pipeline: [] + saveResultAsEntity: changeStream + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandSucceededEvent: + commandName: aggregate + + - + description: 'collection.insertOne retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [insert] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: insertOne + arguments: + document: { _id: 2, x: 22 } + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandSucceededEvent: + commandName: insert + + - + description: 'collection.insertMany retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [insert] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: insertMany + arguments: + documents: + - { _id: 2, x: 22 } + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandSucceededEvent: + commandName: insert + + - + description: 'collection.deleteOne retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [delete] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: deleteOne + arguments: + filter: {} + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandSucceededEvent: + commandName: delete + + - + description: 'collection.deleteMany retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [delete] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: deleteMany + arguments: + filter: {} + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandSucceededEvent: + commandName: delete + + - + description: 'collection.replaceOne retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [update] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: replaceOne + arguments: + filter: {} + replacement: { x: 22 } + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandSucceededEvent: + commandName: update + + - + description: 'collection.updateOne retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [update] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: updateOne + arguments: + filter: {} + update: { $set: { x: 22 } } + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandSucceededEvent: + commandName: update + + - + description: 'collection.updateMany retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [update] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: updateMany + arguments: + filter: {} + update: { $set: { x: 22 } } + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandSucceededEvent: + commandName: update + + - + description: 'collection.findOneAndDelete retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [findAndModify] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: findOneAndDelete + arguments: + filter: {} + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandSucceededEvent: + commandName: findAndModify + + - + description: 'collection.findOneAndReplace retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [findAndModify] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: findOneAndReplace + arguments: + filter: {} + replacement: { x: 22 } + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandSucceededEvent: + commandName: findAndModify + + - + description: 'collection.findOneAndUpdate retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [findAndModify] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: findOneAndUpdate + arguments: + filter: {} + update: { $set: { x: 22 } } + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandSucceededEvent: + commandName: findAndModify + + - + description: 'collection.bulkWrite retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [insert] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: bulkWrite + arguments: + requests: + - insertOne: + document: { _id: 2, x: 22 } + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandSucceededEvent: + commandName: insert + + - + description: 'collection.createIndex retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [createIndexes] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: createIndex + arguments: + keys: { x: 11 } + name: "x_11" + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: createIndexes + - commandFailedEvent: + commandName: createIndexes + - commandStartedEvent: + commandName: createIndexes + - commandFailedEvent: + commandName: createIndexes + - commandStartedEvent: + commandName: createIndexes + - commandFailedEvent: + commandName: createIndexes + - commandStartedEvent: + commandName: createIndexes + - commandSucceededEvent: + commandName: createIndexes + + - + description: 'collection.dropIndex retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + - + object: *utilCollection + name: createIndex + arguments: + keys: { x: 11 } + name: "x_11" + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [dropIndexes] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: dropIndex + arguments: + name: "x_11" + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandSucceededEvent: + commandName: dropIndexes + + - + description: 'collection.dropIndexes retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [dropIndexes] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: dropIndexes + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandSucceededEvent: + commandName: dropIndexes diff --git a/test/spec/client-backpressure/backpressure-retry-loop.yml.template b/test/spec/client-backpressure/backpressure-retry-loop.yml.template new file mode 100644 index 0000000000..049cbbac3a --- /dev/null +++ b/test/spec/client-backpressure/backpressure-retry-loop.yml.template @@ -0,0 +1,128 @@ +# Tests in this file are generated from backpressure-retry-loop.yml.template. + +description: tests that operations respect overload backoff retry loop + +schemaVersion: '1.0' + +runOnRequirements: + - + minServerVersion: '4.4' # failCommand + topologies: [replicaset, sharded, load-balanced] + +createEntities: + - + client: + id: &client client + useMultipleMongoses: false + observeEvents: [ 'commandStartedEvent', 'commandSucceededEvent', 'commandFailedEvent' ] + + - + client: + id: &failPointClient failPointClient + useMultipleMongoses: false + + - + database: + id: &utilDb utilDb + client: *failPointClient + databaseName: &database_name retryable-writes-tests + + - + collection: + id: &utilCollection utilCollection + database: *utilDb + collectionName: &collection_name coll + + - + database: + id: &database database + client: *client + databaseName: &database_name retryable-writes-tests + - + collection: + id: &collection collection + database: *database + collectionName: &collection_name coll + +initialData: + - + collectionName: *collection_name + databaseName: *database_name + documents: + - { _id: 1, x: 11 } + - { _id: 2, x: 22 } + +tests: +{% for operation in operations %} + - + description: '{{operation.object}}.{{operation.operation_name}} retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + {%- if operation.operation_name == "dropIndex" %} + - + object: *utilCollection + name: createIndex + arguments: + keys: { x: 11 } + name: "x_11" + {%- endif %} + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [{{operation.command_name}}] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *{{operation.object}} + name: {{operation.operation_name}} + {%- if operation.arguments|length > 0 %} + arguments: + {%- for arg in operation.arguments %} + {{arg}} + {%- endfor -%} + {%- endif %} + {%- if operation.operation_name == "createChangeStream" %} + saveResultAsEntity: changeStream + {%- endif %} + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: {{operation.command_name}} + - commandFailedEvent: + commandName: {{operation.command_name}} + - commandStartedEvent: + commandName: {{operation.command_name}} + - commandFailedEvent: + commandName: {{operation.command_name}} + - commandStartedEvent: + commandName: {{operation.command_name}} + - commandFailedEvent: + commandName: {{operation.command_name}} + - commandStartedEvent: + commandName: {{operation.command_name}} + - commandSucceededEvent: + commandName: {{operation.command_name}} +{% endfor -%} diff --git a/test/spec/client-backpressure/backpressure-retry-max-attempts.json b/test/spec/client-backpressure/backpressure-retry-max-attempts.json new file mode 100644 index 0000000000..36afc55cc5 --- /dev/null +++ b/test/spec/client-backpressure/backpressure-retry-max-attempts.json @@ -0,0 +1,1262 @@ +{ + "description": "tests that operations retry at most maxAttempts=5 times", + "schemaVersion": "1.0", + "runOnRequirements": [ + { + "minServerVersion": "4.4", + "topologies": [ + "replicaset", + "sharded", + "load-balanced" + ] + } + ], + "createEntities": [ + { + "client": { + "id": "client", + "useMultipleMongoses": false + } + }, + { + "client": { + "id": "failPointClient", + "useMultipleMongoses": false + } + }, + { + "database": { + "id": "database", + "client": "client", + "databaseName": "retryable-writes-tests" + } + }, + { + "collection": { + "id": "collection", + "database": "database", + "collectionName": "coll" + } + } + ], + "initialData": [ + { + "collectionName": "coll", + "databaseName": "retryable-writes-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + } + ] + } + ], + "tests": [ + { + "description": "client.listDatabases retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "listDatabases" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "client", + "name": "listDatabases", + "arguments": { + "filter": {} + }, + "expectError": true + } + ] + }, + { + "description": "client.listDatabaseNames retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "listDatabases" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "client", + "name": "listDatabaseNames", + "expectError": true + } + ] + }, + { + "description": "client.createChangeStream retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "client", + "name": "createChangeStream", + "arguments": { + "pipeline": [] + }, + "saveResultAsEntity": "changeStream", + "expectError": true + } + ] + }, + { + "description": "client.clientBulkWrite retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "bulkWrite" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "client", + "name": "clientBulkWrite", + "arguments": { + "models": [ + { + "insertOne": { + "namespace": "retryable-writes-tests.coll", + "document": { + "_id": 8, + "x": 88 + } + } + } + ] + }, + "expectError": true + } + ] + }, + { + "description": "database.aggregate retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "database", + "name": "aggregate", + "arguments": { + "pipeline": [ + { + "$listLocalSessions": {} + }, + { + "$limit": 1 + } + ] + }, + "expectError": true + } + ] + }, + { + "description": "database.listCollections retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "listCollections" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "database", + "name": "listCollections", + "arguments": { + "filter": {} + }, + "expectError": true + } + ] + }, + { + "description": "database.listCollectionNames retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "listCollections" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "database", + "name": "listCollectionNames", + "arguments": { + "filter": {} + }, + "expectError": true + } + ] + }, + { + "description": "database.runCommand retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "ping" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "database", + "name": "runCommand", + "arguments": { + "command": { + "ping": 1 + }, + "commandName": "ping" + }, + "expectError": true + } + ] + }, + { + "description": "database.createChangeStream retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "database", + "name": "createChangeStream", + "arguments": { + "pipeline": [] + }, + "saveResultAsEntity": "changeStream", + "expectError": true + } + ] + }, + { + "description": "collection.aggregate retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "aggregate", + "arguments": { + "pipeline": [] + }, + "expectError": true + } + ] + }, + { + "description": "collection.countDocuments retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "countDocuments", + "arguments": { + "filter": {} + }, + "expectError": true + } + ] + }, + { + "description": "collection.estimatedDocumentCount retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "count" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "estimatedDocumentCount", + "expectError": true + } + ] + }, + { + "description": "collection.distinct retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "distinct" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "distinct", + "arguments": { + "fieldName": "x", + "filter": {} + }, + "expectError": true + } + ] + }, + { + "description": "collection.find retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "find" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "find", + "arguments": { + "filter": {} + }, + "expectError": true + } + ] + }, + { + "description": "collection.findOne retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "find" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "findOne", + "arguments": { + "filter": {} + }, + "expectError": true + } + ] + }, + { + "description": "collection.listIndexes retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "listIndexes" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "listIndexes", + "expectError": true + } + ] + }, + { + "description": "collection.listIndexNames retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "listIndexes" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "listIndexNames", + "expectError": true + } + ] + }, + { + "description": "collection.createChangeStream retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "createChangeStream", + "arguments": { + "pipeline": [] + }, + "saveResultAsEntity": "changeStream", + "expectError": true + } + ] + }, + { + "description": "collection.insertOne retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "insert" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "insertOne", + "arguments": { + "document": { + "_id": 2, + "x": 22 + } + }, + "expectError": true + } + ] + }, + { + "description": "collection.insertMany retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "insert" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "insertMany", + "arguments": { + "documents": [ + { + "_id": 2, + "x": 22 + } + ] + }, + "expectError": true + } + ] + }, + { + "description": "collection.deleteOne retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "delete" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "deleteOne", + "arguments": { + "filter": {} + }, + "expectError": true + } + ] + }, + { + "description": "collection.deleteMany retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "delete" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "deleteMany", + "arguments": { + "filter": {} + }, + "expectError": true + } + ] + }, + { + "description": "collection.replaceOne retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "update" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "replaceOne", + "arguments": { + "filter": {}, + "replacement": { + "x": 22 + } + }, + "expectError": true + } + ] + }, + { + "description": "collection.updateOne retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "update" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "updateOne", + "arguments": { + "filter": {}, + "update": { + "$set": { + "x": 22 + } + } + }, + "expectError": true + } + ] + }, + { + "description": "collection.updateMany retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "update" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "updateMany", + "arguments": { + "filter": {}, + "update": { + "$set": { + "x": 22 + } + } + }, + "expectError": true + } + ] + }, + { + "description": "collection.findOneAndDelete retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "findAndModify" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "findOneAndDelete", + "arguments": { + "filter": {} + }, + "expectError": true + } + ] + }, + { + "description": "collection.findOneAndReplace retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "findAndModify" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "findOneAndReplace", + "arguments": { + "filter": {}, + "replacement": { + "x": 22 + } + }, + "expectError": true + } + ] + }, + { + "description": "collection.findOneAndUpdate retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "findAndModify" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "findOneAndUpdate", + "arguments": { + "filter": {}, + "update": { + "$set": { + "x": 22 + } + } + }, + "expectError": true + } + ] + }, + { + "description": "collection.bulkWrite retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "insert" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "bulkWrite", + "arguments": { + "requests": [ + { + "insertOne": { + "document": { + "_id": 2, + "x": 22 + } + } + } + ] + }, + "expectError": true + } + ] + }, + { + "description": "collection.createIndex retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "createIndexes" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "createIndex", + "arguments": { + "keys": { + "x": 11 + }, + "name": "x_11" + }, + "expectError": true + } + ] + }, + { + "description": "collection.dropIndex retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "dropIndexes" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "dropIndex", + "arguments": { + "name": "x_11" + }, + "expectError": true + } + ] + }, + { + "description": "collection.dropIndexes retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "dropIndexes" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "dropIndexes", + "expectError": true + } + ] + } + ] +} diff --git a/test/spec/client-backpressure/backpressure-retry-max-attempts.yml b/test/spec/client-backpressure/backpressure-retry-max-attempts.yml new file mode 100644 index 0000000000..998b48c78e --- /dev/null +++ b/test/spec/client-backpressure/backpressure-retry-max-attempts.yml @@ -0,0 +1,753 @@ +# Tests in this file are generated from backpressure-retry-loop.yml.template. + +description: tests that operations retry at most maxAttempts=5 times + +schemaVersion: '1.0' + +runOnRequirements: + - + minServerVersion: '4.4' # failCommand + topologies: [replicaset, sharded, load-balanced] + +createEntities: + - + client: + id: &client client + useMultipleMongoses: false + + - + client: + id: &failPointClient failPointClient + useMultipleMongoses: false + + - + database: + id: &database database + client: *client + databaseName: &database_name retryable-writes-tests + - + collection: + id: &collection collection + database: *database + collectionName: &collection_name coll + +initialData: + - + collectionName: *collection_name + databaseName: *database_name + documents: + - { _id: 1, x: 11 } + - { _id: 2, x: 22 } + +tests: + + - + description: 'client.listDatabases retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [listDatabases] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *client + name: listDatabases + arguments: + filter: {} + expectError: true + + - + description: 'client.listDatabaseNames retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [listDatabases] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *client + name: listDatabaseNames + expectError: true + + - + description: 'client.createChangeStream retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [aggregate] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *client + name: createChangeStream + arguments: + pipeline: [] + saveResultAsEntity: changeStream + expectError: true + + - + description: 'client.clientBulkWrite retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [bulkWrite] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *client + name: clientBulkWrite + arguments: + models: + - insertOne: + namespace: retryable-writes-tests.coll + document: { _id: 8, x: 88 } + expectError: true + + - + description: 'database.aggregate retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [aggregate] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *database + name: aggregate + arguments: + pipeline: [ { $listLocalSessions: {} }, { $limit: 1 } ] + expectError: true + + - + description: 'database.listCollections retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [listCollections] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *database + name: listCollections + arguments: + filter: {} + expectError: true + + - + description: 'database.listCollectionNames retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [listCollections] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *database + name: listCollectionNames + arguments: + filter: {} + expectError: true + + - + description: 'database.runCommand retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [ping] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *database + name: runCommand + arguments: + command: { ping: 1 } + commandName: ping + expectError: true + + - + description: 'database.createChangeStream retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [aggregate] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *database + name: createChangeStream + arguments: + pipeline: [] + saveResultAsEntity: changeStream + expectError: true + + - + description: 'collection.aggregate retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [aggregate] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: aggregate + arguments: + pipeline: [] + expectError: true + + - + description: 'collection.countDocuments retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [aggregate] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: countDocuments + arguments: + filter: {} + expectError: true + + - + description: 'collection.estimatedDocumentCount retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [count] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: estimatedDocumentCount + expectError: true + + - + description: 'collection.distinct retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [distinct] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: distinct + arguments: + fieldName: x + filter: {} + expectError: true + + - + description: 'collection.find retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [find] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: find + arguments: + filter: {} + expectError: true + + - + description: 'collection.findOne retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [find] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: findOne + arguments: + filter: {} + expectError: true + + - + description: 'collection.listIndexes retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [listIndexes] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: listIndexes + expectError: true + + - + description: 'collection.listIndexNames retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [listIndexes] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: listIndexNames + expectError: true + + - + description: 'collection.createChangeStream retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [aggregate] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: createChangeStream + arguments: + pipeline: [] + saveResultAsEntity: changeStream + expectError: true + + - + description: 'collection.insertOne retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [insert] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: insertOne + arguments: + document: { _id: 2, x: 22 } + expectError: true + + - + description: 'collection.insertMany retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [insert] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: insertMany + arguments: + documents: + - { _id: 2, x: 22 } + expectError: true + + - + description: 'collection.deleteOne retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [delete] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: deleteOne + arguments: + filter: {} + expectError: true + + - + description: 'collection.deleteMany retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [delete] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: deleteMany + arguments: + filter: {} + expectError: true + + - + description: 'collection.replaceOne retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [update] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: replaceOne + arguments: + filter: {} + replacement: { x: 22 } + expectError: true + + - + description: 'collection.updateOne retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [update] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: updateOne + arguments: + filter: {} + update: { $set: { x: 22 } } + expectError: true + + - + description: 'collection.updateMany retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [update] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: updateMany + arguments: + filter: {} + update: { $set: { x: 22 } } + expectError: true + + - + description: 'collection.findOneAndDelete retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [findAndModify] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: findOneAndDelete + arguments: + filter: {} + expectError: true + + - + description: 'collection.findOneAndReplace retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [findAndModify] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: findOneAndReplace + arguments: + filter: {} + replacement: { x: 22 } + expectError: true + + - + description: 'collection.findOneAndUpdate retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [findAndModify] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: findOneAndUpdate + arguments: + filter: {} + update: { $set: { x: 22 } } + expectError: true + + - + description: 'collection.bulkWrite retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [insert] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: bulkWrite + arguments: + requests: + - insertOne: + document: { _id: 2, x: 22 } + expectError: true + + - + description: 'collection.createIndex retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [createIndexes] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: createIndex + arguments: + keys: { x: 11 } + name: "x_11" + expectError: true + + - + description: 'collection.dropIndex retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [dropIndexes] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: dropIndex + arguments: + name: "x_11" + expectError: true + + - + description: 'collection.dropIndexes retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [dropIndexes] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: dropIndexes + expectError: true diff --git a/test/spec/client-backpressure/backpressure-retry-max-attempts.yml.template b/test/spec/client-backpressure/backpressure-retry-max-attempts.yml.template new file mode 100644 index 0000000000..bf089211fd --- /dev/null +++ b/test/spec/client-backpressure/backpressure-retry-max-attempts.yml.template @@ -0,0 +1,72 @@ +# Tests in this file are generated from backpressure-retry-max-attempts.yml.template. + +description: tests that operations retry at most maxAttempts=5 times + +schemaVersion: '1.0' + +runOnRequirements: + - + minServerVersion: '4.4' # failCommand + topologies: [replicaset, sharded, load-balanced] + +createEntities: + - + client: + id: &client client + useMultipleMongoses: false + + - + client: + id: &failPointClient failPointClient + useMultipleMongoses: false + + - + database: + id: &database database + client: *client + databaseName: &database_name retryable-writes-tests + - + collection: + id: &collection collection + database: *database + collectionName: &collection_name coll + +initialData: + - + collectionName: *collection_name + databaseName: *database_name + documents: + - { _id: 1, x: 11 } + - { _id: 2, x: 22 } + +tests: +{% for operation in operations %} + - + description: '{{operation.object}}.{{operation.operation_name}} retries at most maxAttempts=5 times' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [{{operation.command_name}}] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *{{operation.object}} + name: {{operation.operation_name}} + {%- if operation.arguments|length > 0 %} + arguments: + {%- for arg in operation.arguments %} + {{arg}} + {%- endfor -%} + {%- endif %} + {%- if operation.operation_name == "createChangeStream" %} + saveResultAsEntity: changeStream + {%- endif %} + expectError: true +{% endfor -%} From db396757a1dba948ef3f7488b3b36e5681523c52 Mon Sep 17 00:00:00 2001 From: bailey Date: Mon, 1 Dec 2025 16:41:03 -0700 Subject: [PATCH 08/10] add prose tests and latest versions of tests --- src/operations/execute_operation.ts | 2 +- .../client-backpressure.prose.test.ts | 60 + .../client-backpressure.spec.test.ts | 2 +- test/spec/client-backpressure/README.md | 45 + .../backpressure-retry-loop.json | 3 + .../backpressure-retry-loop.yml | 2 + .../backpressure-retry-loop.yml.template | 4 + .../backpressure-retry-max-attempts.json | 2408 +++++++++++++++-- .../backpressure-retry-max-attempts.yml | 1125 +++++++- ...ckpressure-retry-max-attempts.yml.template | 38 +- 10 files changed, 3461 insertions(+), 228 deletions(-) create mode 100644 test/integration/client-backpressure/client-backpressure.prose.test.ts diff --git a/src/operations/execute_operation.ts b/src/operations/execute_operation.ts index 6094c86b75..db4ac49ced 100644 --- a/src/operations/execute_operation.ts +++ b/src/operations/execute_operation.ts @@ -280,7 +280,7 @@ async function executeOperationWithRetries< } // if we have exhausted overload retry attempts, throw. - if (systemOverloadRetryAttempt >= maxSystemOverloadRetryAttempts) { + if (systemOverloadRetryAttempt > maxSystemOverloadRetryAttempts) { throw previousOperationError; } diff --git a/test/integration/client-backpressure/client-backpressure.prose.test.ts b/test/integration/client-backpressure/client-backpressure.prose.test.ts new file mode 100644 index 0000000000..56d274fffe --- /dev/null +++ b/test/integration/client-backpressure/client-backpressure.prose.test.ts @@ -0,0 +1,60 @@ +import { expect } from 'chai'; +import * as sinon from 'sinon'; + +import { type Collection, type MongoClient, MongoServerError } from '../../../src'; +import { clearFailPoint, configureFailPoint, measureDuration } from '../../tools/utils'; + +describe('Client Backpressure (Prose)', function () { + let client: MongoClient; + let collection: Collection; + + beforeEach(async function () { + client = this.configuration.newClient(); + await client.connect(); + + collection = client.db('foo').collection('bar'); + }); + + afterEach(async function () { + await client.close(); + await clearFailPoint(this.configuration); + }); + + it( + 'Test 1: Operation Retry Uses Exponential Backoff', + { + requires: { + mongodb: '4.4' + } + }, + async function () { + await configureFailPoint(this.configuration, { + configureFailPoint: 'failCommand', + mode: 'alwaysOn', + data: { + failCommands: ['insert'], + errorCode: 2, + errorLabels: ['SystemOverloadedError', 'RetryableError'] + } + }); + + const stub = sinon.stub(Math, 'random'); + + stub.returns(0); + + const { duration: durationNoBackoff } = await measureDuration(async () => { + const error = await collection.insertOne({ a: 1 }).catch(e => e); + expect(error).to.be.instanceof(MongoServerError); + }); + + stub.returns(1); + + const { duration: durationBackoff } = await measureDuration(async () => { + const error = await collection.insertOne({ a: 1 }).catch(e => e); + expect(error).to.be.instanceof(MongoServerError); + }); + + expect(durationBackoff - durationNoBackoff).to.be.within(3100 - 1000, 3100 + 1000); + } + ); +}); diff --git a/test/integration/client-backpressure/client-backpressure.spec.test.ts b/test/integration/client-backpressure/client-backpressure.spec.test.ts index 4175929252..9485c46a96 100644 --- a/test/integration/client-backpressure/client-backpressure.spec.test.ts +++ b/test/integration/client-backpressure/client-backpressure.spec.test.ts @@ -3,7 +3,7 @@ import { runUnifiedSuite } from '../../tools/unified-spec-runner/runner'; import { type Test } from '../../tools/unified-spec-runner/schema'; const skippedTests = { - 'collection.dropIndexes retries at most maxAttempts times (maxAttempts=5)': + 'collection.dropIndexes retries at most maxAttempts=5 times': 'TODO(NODE-6517): dropIndexes squashes all errors other than ns not found' }; diff --git a/test/spec/client-backpressure/README.md b/test/spec/client-backpressure/README.md index 7a70e4ad76..26d664f13e 100644 --- a/test/spec/client-backpressure/README.md +++ b/test/spec/client-backpressure/README.md @@ -10,6 +10,51 @@ retryable reads. These tests utilize the [Unified Test Format](../../unified-tes Several prose tests, which are not easily expressed in YAML, are also presented in this file. Those tests will need to be manually implemented by each driver. +### Prose Tests + +#### Test 1: Operation Retry Uses Exponential Backoff + +Drivers should test that retries do not occur immediately when a SystemOverloadedError is encountered. + +1. let `client` be a `MongoClient` +2. let `collection` be a collection +3. Now, run transactions without backoff: + 1. Configure the random number generator used for jitter to always return `0` -- this effectively disables backoff. + + 2. Configure the following failPoint: + + ```javascript + { + configureFailPoint: 'failCommand', + mode: 'alwaysOn', + data: { + failCommands: ['insert'], + errorCode: 2, + errorLabels: ['SystemOverloadedError', 'RetryableError'] + } + } + ``` + + 3. Execute the following command. Expect that the command errors. Measure the duration of the command execution. + + ```javascript + const start = performance.now(); + expect( + await coll.insertOne({ a: 1 }).catch(e => e) + ).to.be.an.instanceof(MongoServerError); + const end = performance.now(); + ``` + + 4. Configure the random number generator used for jitter to always return `1`. + 5. Execute step 3 again. + 6. Compare the two time between the two runs. + ```python + assertTrue(absolute_value(with_backoff_time - (no_backoff_time + 3.1 seconds)) < 1) + ``` + The sum of 5 backoffs is 3.1 seconds. There is a 1-second window to account for potential variance between + the two runs. + + ## Changelog - 2025-XX-XX: Initial version. diff --git a/test/spec/client-backpressure/backpressure-retry-loop.json b/test/spec/client-backpressure/backpressure-retry-loop.json index 21fc802344..ae944f73ad 100644 --- a/test/spec/client-backpressure/backpressure-retry-loop.json +++ b/test/spec/client-backpressure/backpressure-retry-loop.json @@ -381,6 +381,9 @@ }, { "description": "client.clientBulkWrite retries using operation loop", + "runOnRequirements": { + "minServerVersion": "8.0" + }, "operations": [ { "object": "utilCollection", diff --git a/test/spec/client-backpressure/backpressure-retry-loop.yml b/test/spec/client-backpressure/backpressure-retry-loop.yml index ec2fa25b8f..c612986233 100644 --- a/test/spec/client-backpressure/backpressure-retry-loop.yml +++ b/test/spec/client-backpressure/backpressure-retry-loop.yml @@ -223,6 +223,8 @@ tests: - description: 'client.clientBulkWrite retries using operation loop' + runOnRequirements: + minServerVersion: '8.0' operations: - object: *utilCollection diff --git a/test/spec/client-backpressure/backpressure-retry-loop.yml.template b/test/spec/client-backpressure/backpressure-retry-loop.yml.template index 049cbbac3a..cad83625d3 100644 --- a/test/spec/client-backpressure/backpressure-retry-loop.yml.template +++ b/test/spec/client-backpressure/backpressure-retry-loop.yml.template @@ -56,6 +56,10 @@ tests: {% for operation in operations %} - description: '{{operation.object}}.{{operation.operation_name}} retries using operation loop' + {%- if ((operation.operation_name == 'clientBulkWrite')) %} + runOnRequirements: + minServerVersion: '8.0' + {%- endif %} operations: - object: *utilCollection diff --git a/test/spec/client-backpressure/backpressure-retry-max-attempts.json b/test/spec/client-backpressure/backpressure-retry-max-attempts.json index 36afc55cc5..7e9cc67d7a 100644 --- a/test/spec/client-backpressure/backpressure-retry-max-attempts.json +++ b/test/spec/client-backpressure/backpressure-retry-max-attempts.json @@ -15,7 +15,12 @@ { "client": { "id": "client", - "useMultipleMongoses": false + "useMultipleMongoses": false, + "observeEvents": [ + "commandStartedEvent", + "commandSucceededEvent", + "commandFailedEvent" + ] } }, { @@ -57,7 +62,7 @@ ], "tests": [ { - "description": "client.listDatabases retries at most maxAttempts times (maxAttempts=5)", + "description": "client.listDatabases retries at most maxAttempts=5 times", "operations": [ { "name": "failPoint", @@ -66,9 +71,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "listDatabases" @@ -90,10 +93,77 @@ }, "expectError": true } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + } + ] + } ] }, { - "description": "client.listDatabaseNames retries at most maxAttempts times (maxAttempts=5)", + "description": "client.listDatabaseNames retries at most maxAttempts=5 times", "operations": [ { "name": "failPoint", @@ -102,9 +172,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "listDatabases" @@ -123,10 +191,77 @@ "name": "listDatabaseNames", "expectError": true } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + } + ] + } ] }, { - "description": "client.createChangeStream retries at most maxAttempts times (maxAttempts=5)", + "description": "client.createChangeStream retries at most maxAttempts=5 times", "operations": [ { "name": "failPoint", @@ -135,9 +270,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "aggregate" @@ -160,10 +293,80 @@ "saveResultAsEntity": "changeStream", "expectError": true } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + } + ] + } ] }, { - "description": "client.clientBulkWrite retries at most maxAttempts times (maxAttempts=5)", + "description": "client.clientBulkWrite retries at most maxAttempts=5 times", + "runOnRequirements": { + "minServerVersion": "8.0" + }, "operations": [ { "name": "failPoint", @@ -172,9 +375,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "bulkWrite" @@ -206,10 +407,77 @@ }, "expectError": true } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandFailedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandStartedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandFailedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandStartedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandFailedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandStartedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandFailedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandStartedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandFailedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandStartedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandFailedEvent": { + "commandName": "bulkWrite" + } + } + ] + } ] }, { - "description": "database.aggregate retries at most maxAttempts times (maxAttempts=5)", + "description": "database.aggregate retries at most maxAttempts=5 times", "operations": [ { "name": "failPoint", @@ -218,9 +486,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "aggregate" @@ -249,10 +515,77 @@ }, "expectError": true } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + } + ] + } ] }, { - "description": "database.listCollections retries at most maxAttempts times (maxAttempts=5)", + "description": "database.listCollections retries at most maxAttempts=5 times", "operations": [ { "name": "failPoint", @@ -261,9 +594,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "listCollections" @@ -285,10 +616,77 @@ }, "expectError": true } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + } + ] + } ] }, { - "description": "database.listCollectionNames retries at most maxAttempts times (maxAttempts=5)", + "description": "database.listCollectionNames retries at most maxAttempts=5 times", "operations": [ { "name": "failPoint", @@ -297,9 +695,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "listCollections" @@ -321,10 +717,77 @@ }, "expectError": true } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + } + ] + } ] }, { - "description": "database.runCommand retries at most maxAttempts times (maxAttempts=5)", + "description": "database.runCommand retries at most maxAttempts=5 times", "operations": [ { "name": "failPoint", @@ -333,9 +796,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "ping" @@ -360,10 +821,77 @@ }, "expectError": true } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "ping" + } + }, + { + "commandFailedEvent": { + "commandName": "ping" + } + }, + { + "commandStartedEvent": { + "commandName": "ping" + } + }, + { + "commandFailedEvent": { + "commandName": "ping" + } + }, + { + "commandStartedEvent": { + "commandName": "ping" + } + }, + { + "commandFailedEvent": { + "commandName": "ping" + } + }, + { + "commandStartedEvent": { + "commandName": "ping" + } + }, + { + "commandFailedEvent": { + "commandName": "ping" + } + }, + { + "commandStartedEvent": { + "commandName": "ping" + } + }, + { + "commandFailedEvent": { + "commandName": "ping" + } + }, + { + "commandStartedEvent": { + "commandName": "ping" + } + }, + { + "commandFailedEvent": { + "commandName": "ping" + } + } + ] + } ] }, { - "description": "database.createChangeStream retries at most maxAttempts times (maxAttempts=5)", + "description": "database.createChangeStream retries at most maxAttempts=5 times", "operations": [ { "name": "failPoint", @@ -372,9 +900,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "aggregate" @@ -397,10 +923,77 @@ "saveResultAsEntity": "changeStream", "expectError": true } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + } + ] + } ] }, { - "description": "collection.aggregate retries at most maxAttempts times (maxAttempts=5)", + "description": "collection.aggregate retries at most maxAttempts=5 times", "operations": [ { "name": "failPoint", @@ -409,9 +1002,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "aggregate" @@ -433,29 +1024,94 @@ }, "expectError": true } - ] - }, - { - "description": "collection.countDocuments retries at most maxAttempts times (maxAttempts=5)", - "operations": [ - { - "name": "failPoint", - "object": "testRunner", - "arguments": { - "client": "failPointClient", - "failPoint": { - "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, - "data": { - "failCommands": [ - "aggregate" - ], - "errorLabels": [ - "RetryableError", - "SystemOverloadedError" - ], + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + } + ] + } + ] + }, + { + "description": "collection.countDocuments retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], "errorCode": 2 } } @@ -469,10 +1125,77 @@ }, "expectError": true } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + } + ] + } ] }, { - "description": "collection.estimatedDocumentCount retries at most maxAttempts times (maxAttempts=5)", + "description": "collection.estimatedDocumentCount retries at most maxAttempts=5 times", "operations": [ { "name": "failPoint", @@ -481,9 +1204,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "count" @@ -502,10 +1223,77 @@ "name": "estimatedDocumentCount", "expectError": true } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "count" + } + }, + { + "commandFailedEvent": { + "commandName": "count" + } + }, + { + "commandStartedEvent": { + "commandName": "count" + } + }, + { + "commandFailedEvent": { + "commandName": "count" + } + }, + { + "commandStartedEvent": { + "commandName": "count" + } + }, + { + "commandFailedEvent": { + "commandName": "count" + } + }, + { + "commandStartedEvent": { + "commandName": "count" + } + }, + { + "commandFailedEvent": { + "commandName": "count" + } + }, + { + "commandStartedEvent": { + "commandName": "count" + } + }, + { + "commandFailedEvent": { + "commandName": "count" + } + }, + { + "commandStartedEvent": { + "commandName": "count" + } + }, + { + "commandFailedEvent": { + "commandName": "count" + } + } + ] + } ] }, { - "description": "collection.distinct retries at most maxAttempts times (maxAttempts=5)", + "description": "collection.distinct retries at most maxAttempts=5 times", "operations": [ { "name": "failPoint", @@ -514,9 +1302,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "distinct" @@ -539,10 +1325,77 @@ }, "expectError": true } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "distinct" + } + }, + { + "commandFailedEvent": { + "commandName": "distinct" + } + }, + { + "commandStartedEvent": { + "commandName": "distinct" + } + }, + { + "commandFailedEvent": { + "commandName": "distinct" + } + }, + { + "commandStartedEvent": { + "commandName": "distinct" + } + }, + { + "commandFailedEvent": { + "commandName": "distinct" + } + }, + { + "commandStartedEvent": { + "commandName": "distinct" + } + }, + { + "commandFailedEvent": { + "commandName": "distinct" + } + }, + { + "commandStartedEvent": { + "commandName": "distinct" + } + }, + { + "commandFailedEvent": { + "commandName": "distinct" + } + }, + { + "commandStartedEvent": { + "commandName": "distinct" + } + }, + { + "commandFailedEvent": { + "commandName": "distinct" + } + } + ] + } ] }, { - "description": "collection.find retries at most maxAttempts times (maxAttempts=5)", + "description": "collection.find retries at most maxAttempts=5 times", "operations": [ { "name": "failPoint", @@ -551,9 +1404,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "find" @@ -575,10 +1426,77 @@ }, "expectError": true } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + } + ] + } ] }, { - "description": "collection.findOne retries at most maxAttempts times (maxAttempts=5)", + "description": "collection.findOne retries at most maxAttempts=5 times", "operations": [ { "name": "failPoint", @@ -587,9 +1505,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "find" @@ -611,10 +1527,77 @@ }, "expectError": true } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + } + ] + } ] }, { - "description": "collection.listIndexes retries at most maxAttempts times (maxAttempts=5)", + "description": "collection.listIndexes retries at most maxAttempts=5 times", "operations": [ { "name": "failPoint", @@ -623,9 +1606,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "listIndexes" @@ -644,10 +1625,77 @@ "name": "listIndexes", "expectError": true } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + } + ] + } ] }, { - "description": "collection.listIndexNames retries at most maxAttempts times (maxAttempts=5)", + "description": "collection.listIndexNames retries at most maxAttempts=5 times", "operations": [ { "name": "failPoint", @@ -656,9 +1704,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "listIndexes" @@ -677,10 +1723,77 @@ "name": "listIndexNames", "expectError": true } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + } + ] + } ] }, { - "description": "collection.createChangeStream retries at most maxAttempts times (maxAttempts=5)", + "description": "collection.createChangeStream retries at most maxAttempts=5 times", "operations": [ { "name": "failPoint", @@ -689,9 +1802,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "aggregate" @@ -714,10 +1825,77 @@ "saveResultAsEntity": "changeStream", "expectError": true } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + } + ] + } ] }, { - "description": "collection.insertOne retries at most maxAttempts times (maxAttempts=5)", + "description": "collection.insertOne retries at most maxAttempts=5 times", "operations": [ { "name": "failPoint", @@ -726,9 +1904,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "insert" @@ -753,10 +1929,77 @@ }, "expectError": true } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + } + ] + } ] }, { - "description": "collection.insertMany retries at most maxAttempts times (maxAttempts=5)", + "description": "collection.insertMany retries at most maxAttempts=5 times", "operations": [ { "name": "failPoint", @@ -765,9 +2008,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "insert" @@ -794,10 +2035,77 @@ }, "expectError": true } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + } + ] + } ] }, { - "description": "collection.deleteOne retries at most maxAttempts times (maxAttempts=5)", + "description": "collection.deleteOne retries at most maxAttempts=5 times", "operations": [ { "name": "failPoint", @@ -806,9 +2114,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "delete" @@ -830,10 +2136,77 @@ }, "expectError": true } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + } + ] + } ] }, { - "description": "collection.deleteMany retries at most maxAttempts times (maxAttempts=5)", + "description": "collection.deleteMany retries at most maxAttempts=5 times", "operations": [ { "name": "failPoint", @@ -842,9 +2215,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "delete" @@ -866,10 +2237,77 @@ }, "expectError": true } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + } + ] + } ] }, { - "description": "collection.replaceOne retries at most maxAttempts times (maxAttempts=5)", + "description": "collection.replaceOne retries at most maxAttempts=5 times", "operations": [ { "name": "failPoint", @@ -878,9 +2316,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "update" @@ -892,23 +2328,90 @@ "errorCode": 2 } } - } - }, - { - "object": "collection", - "name": "replaceOne", - "arguments": { - "filter": {}, - "replacement": { - "x": 22 - } - }, - "expectError": true + } + }, + { + "object": "collection", + "name": "replaceOne", + "arguments": { + "filter": {}, + "replacement": { + "x": 22 + } + }, + "expectError": true + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + } + ] } ] }, { - "description": "collection.updateOne retries at most maxAttempts times (maxAttempts=5)", + "description": "collection.updateOne retries at most maxAttempts=5 times", "operations": [ { "name": "failPoint", @@ -917,9 +2420,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "update" @@ -946,10 +2447,77 @@ }, "expectError": true } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + } + ] + } ] }, { - "description": "collection.updateMany retries at most maxAttempts times (maxAttempts=5)", + "description": "collection.updateMany retries at most maxAttempts=5 times", "operations": [ { "name": "failPoint", @@ -958,9 +2526,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "update" @@ -987,10 +2553,77 @@ }, "expectError": true } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + } + ] + } ] }, { - "description": "collection.findOneAndDelete retries at most maxAttempts times (maxAttempts=5)", + "description": "collection.findOneAndDelete retries at most maxAttempts=5 times", "operations": [ { "name": "failPoint", @@ -999,9 +2632,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "findAndModify" @@ -1023,10 +2654,77 @@ }, "expectError": true } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + } + ] + } ] }, { - "description": "collection.findOneAndReplace retries at most maxAttempts times (maxAttempts=5)", + "description": "collection.findOneAndReplace retries at most maxAttempts=5 times", "operations": [ { "name": "failPoint", @@ -1035,9 +2733,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "findAndModify" @@ -1062,10 +2758,77 @@ }, "expectError": true } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + } + ] + } ] }, { - "description": "collection.findOneAndUpdate retries at most maxAttempts times (maxAttempts=5)", + "description": "collection.findOneAndUpdate retries at most maxAttempts=5 times", "operations": [ { "name": "failPoint", @@ -1074,9 +2837,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "findAndModify" @@ -1103,10 +2864,77 @@ }, "expectError": true } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + } + ] + } ] }, { - "description": "collection.bulkWrite retries at most maxAttempts times (maxAttempts=5)", + "description": "collection.bulkWrite retries at most maxAttempts=5 times", "operations": [ { "name": "failPoint", @@ -1115,9 +2943,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "insert" @@ -1148,10 +2974,77 @@ }, "expectError": true } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + } + ] + } ] }, { - "description": "collection.createIndex retries at most maxAttempts times (maxAttempts=5)", + "description": "collection.createIndex retries at most maxAttempts=5 times", "operations": [ { "name": "failPoint", @@ -1160,9 +3053,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "createIndexes" @@ -1187,10 +3078,77 @@ }, "expectError": true } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "createIndexes" + } + } + ] + } ] }, { - "description": "collection.dropIndex retries at most maxAttempts times (maxAttempts=5)", + "description": "collection.dropIndex retries at most maxAttempts=5 times", "operations": [ { "name": "failPoint", @@ -1199,9 +3157,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "dropIndexes" @@ -1223,10 +3179,77 @@ }, "expectError": true } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + } + ] + } ] }, { - "description": "collection.dropIndexes retries at most maxAttempts times (maxAttempts=5)", + "description": "collection.dropIndexes retries at most maxAttempts=5 times", "operations": [ { "name": "failPoint", @@ -1235,9 +3258,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "dropIndexes" @@ -1256,6 +3277,73 @@ "name": "dropIndexes", "expectError": true } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + } + ] + } ] } ] diff --git a/test/spec/client-backpressure/backpressure-retry-max-attempts.yml b/test/spec/client-backpressure/backpressure-retry-max-attempts.yml index 998b48c78e..9161828e99 100644 --- a/test/spec/client-backpressure/backpressure-retry-max-attempts.yml +++ b/test/spec/client-backpressure/backpressure-retry-max-attempts.yml @@ -1,4 +1,4 @@ -# Tests in this file are generated from backpressure-retry-loop.yml.template. +# Tests in this file are generated from backpressure-retry-max-attempts.yml.template. description: tests that operations retry at most maxAttempts=5 times @@ -14,6 +14,7 @@ createEntities: client: id: &client client useMultipleMongoses: false + observeEvents: [ 'commandStartedEvent', 'commandSucceededEvent', 'commandFailedEvent' ] - client: @@ -42,7 +43,7 @@ initialData: tests: - - description: 'client.listDatabases retries at most maxAttempts times (maxAttempts=5)' + description: 'client.listDatabases retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -50,7 +51,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [listDatabases] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -63,8 +64,39 @@ tests: filter: {} expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + + - - description: 'client.listDatabaseNames retries at most maxAttempts times (maxAttempts=5)' + description: 'client.listDatabaseNames retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -72,7 +104,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [listDatabases] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -83,8 +115,39 @@ tests: name: listDatabaseNames expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + + - - description: 'client.createChangeStream retries at most maxAttempts times (maxAttempts=5)' + description: 'client.createChangeStream retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -92,7 +155,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [aggregate] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -106,8 +169,41 @@ tests: saveResultAsEntity: changeStream expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + + - - description: 'client.clientBulkWrite retries at most maxAttempts times (maxAttempts=5)' + description: 'client.clientBulkWrite retries at most maxAttempts=5 times' + runOnRequirements: + minServerVersion: '8.0' operations: - name: failPoint object: testRunner @@ -115,7 +211,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [bulkWrite] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -131,8 +227,39 @@ tests: document: { _id: 8, x: 88 } expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: bulkWrite + - commandFailedEvent: + commandName: bulkWrite + - commandStartedEvent: + commandName: bulkWrite + - commandFailedEvent: + commandName: bulkWrite + - commandStartedEvent: + commandName: bulkWrite + - commandFailedEvent: + commandName: bulkWrite + - commandStartedEvent: + commandName: bulkWrite + - commandFailedEvent: + commandName: bulkWrite + - commandStartedEvent: + commandName: bulkWrite + - commandFailedEvent: + commandName: bulkWrite + - commandStartedEvent: + commandName: bulkWrite + - commandFailedEvent: + commandName: bulkWrite + + - - description: 'database.aggregate retries at most maxAttempts times (maxAttempts=5)' + description: 'database.aggregate retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -140,7 +267,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [aggregate] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -153,8 +280,39 @@ tests: pipeline: [ { $listLocalSessions: {} }, { $limit: 1 } ] expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + + - - description: 'database.listCollections retries at most maxAttempts times (maxAttempts=5)' + description: 'database.listCollections retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -162,7 +320,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [listCollections] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -175,8 +333,39 @@ tests: filter: {} expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + + - - description: 'database.listCollectionNames retries at most maxAttempts times (maxAttempts=5)' + description: 'database.listCollectionNames retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -184,7 +373,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [listCollections] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -197,8 +386,39 @@ tests: filter: {} expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + + - - description: 'database.runCommand retries at most maxAttempts times (maxAttempts=5)' + description: 'database.runCommand retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -206,7 +426,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [ping] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -220,8 +440,39 @@ tests: commandName: ping expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: ping + - commandFailedEvent: + commandName: ping + - commandStartedEvent: + commandName: ping + - commandFailedEvent: + commandName: ping + - commandStartedEvent: + commandName: ping + - commandFailedEvent: + commandName: ping + - commandStartedEvent: + commandName: ping + - commandFailedEvent: + commandName: ping + - commandStartedEvent: + commandName: ping + - commandFailedEvent: + commandName: ping + - commandStartedEvent: + commandName: ping + - commandFailedEvent: + commandName: ping + + - - description: 'database.createChangeStream retries at most maxAttempts times (maxAttempts=5)' + description: 'database.createChangeStream retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -229,7 +480,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [aggregate] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -243,8 +494,39 @@ tests: saveResultAsEntity: changeStream expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + + - - description: 'collection.aggregate retries at most maxAttempts times (maxAttempts=5)' + description: 'collection.aggregate retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -252,7 +534,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [aggregate] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -265,8 +547,39 @@ tests: pipeline: [] expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + + - - description: 'collection.countDocuments retries at most maxAttempts times (maxAttempts=5)' + description: 'collection.countDocuments retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -274,7 +587,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [aggregate] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -287,8 +600,39 @@ tests: filter: {} expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + + - - description: 'collection.estimatedDocumentCount retries at most maxAttempts times (maxAttempts=5)' + description: 'collection.estimatedDocumentCount retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -296,7 +640,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [count] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -307,8 +651,39 @@ tests: name: estimatedDocumentCount expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: count + - commandFailedEvent: + commandName: count + - commandStartedEvent: + commandName: count + - commandFailedEvent: + commandName: count + - commandStartedEvent: + commandName: count + - commandFailedEvent: + commandName: count + - commandStartedEvent: + commandName: count + - commandFailedEvent: + commandName: count + - commandStartedEvent: + commandName: count + - commandFailedEvent: + commandName: count + - commandStartedEvent: + commandName: count + - commandFailedEvent: + commandName: count + + - - description: 'collection.distinct retries at most maxAttempts times (maxAttempts=5)' + description: 'collection.distinct retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -316,7 +691,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [distinct] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -330,8 +705,39 @@ tests: filter: {} expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: distinct + - commandFailedEvent: + commandName: distinct + - commandStartedEvent: + commandName: distinct + - commandFailedEvent: + commandName: distinct + - commandStartedEvent: + commandName: distinct + - commandFailedEvent: + commandName: distinct + - commandStartedEvent: + commandName: distinct + - commandFailedEvent: + commandName: distinct + - commandStartedEvent: + commandName: distinct + - commandFailedEvent: + commandName: distinct + - commandStartedEvent: + commandName: distinct + - commandFailedEvent: + commandName: distinct + + - - description: 'collection.find retries at most maxAttempts times (maxAttempts=5)' + description: 'collection.find retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -339,7 +745,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [find] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -352,8 +758,39 @@ tests: filter: {} expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + + - - description: 'collection.findOne retries at most maxAttempts times (maxAttempts=5)' + description: 'collection.findOne retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -361,7 +798,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [find] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -374,8 +811,39 @@ tests: filter: {} expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + + - - description: 'collection.listIndexes retries at most maxAttempts times (maxAttempts=5)' + description: 'collection.listIndexes retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -383,7 +851,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [listIndexes] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -394,8 +862,39 @@ tests: name: listIndexes expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + + - - description: 'collection.listIndexNames retries at most maxAttempts times (maxAttempts=5)' + description: 'collection.listIndexNames retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -403,7 +902,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [listIndexes] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -414,8 +913,39 @@ tests: name: listIndexNames expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + + - - description: 'collection.createChangeStream retries at most maxAttempts times (maxAttempts=5)' + description: 'collection.createChangeStream retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -423,7 +953,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [aggregate] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -437,8 +967,39 @@ tests: saveResultAsEntity: changeStream expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + + - - description: 'collection.insertOne retries at most maxAttempts times (maxAttempts=5)' + description: 'collection.insertOne retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -446,7 +1007,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [insert] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -459,8 +1020,39 @@ tests: document: { _id: 2, x: 22 } expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + + - - description: 'collection.insertMany retries at most maxAttempts times (maxAttempts=5)' + description: 'collection.insertMany retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -468,7 +1060,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [insert] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -482,8 +1074,39 @@ tests: - { _id: 2, x: 22 } expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + + - - description: 'collection.deleteOne retries at most maxAttempts times (maxAttempts=5)' + description: 'collection.deleteOne retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -491,7 +1114,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [delete] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -504,8 +1127,39 @@ tests: filter: {} expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + + - - description: 'collection.deleteMany retries at most maxAttempts times (maxAttempts=5)' + description: 'collection.deleteMany retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -513,7 +1167,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [delete] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -526,8 +1180,39 @@ tests: filter: {} expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + + - - description: 'collection.replaceOne retries at most maxAttempts times (maxAttempts=5)' + description: 'collection.replaceOne retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -535,7 +1220,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [update] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -549,8 +1234,39 @@ tests: replacement: { x: 22 } expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + + - - description: 'collection.updateOne retries at most maxAttempts times (maxAttempts=5)' + description: 'collection.updateOne retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -558,7 +1274,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [update] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -572,8 +1288,39 @@ tests: update: { $set: { x: 22 } } expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + + - - description: 'collection.updateMany retries at most maxAttempts times (maxAttempts=5)' + description: 'collection.updateMany retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -581,7 +1328,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [update] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -595,8 +1342,39 @@ tests: update: { $set: { x: 22 } } expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + + - - description: 'collection.findOneAndDelete retries at most maxAttempts times (maxAttempts=5)' + description: 'collection.findOneAndDelete retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -604,7 +1382,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [findAndModify] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -617,8 +1395,39 @@ tests: filter: {} expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + + - - description: 'collection.findOneAndReplace retries at most maxAttempts times (maxAttempts=5)' + description: 'collection.findOneAndReplace retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -626,7 +1435,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [findAndModify] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -640,8 +1449,39 @@ tests: replacement: { x: 22 } expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + + - - description: 'collection.findOneAndUpdate retries at most maxAttempts times (maxAttempts=5)' + description: 'collection.findOneAndUpdate retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -649,7 +1489,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [findAndModify] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -663,8 +1503,39 @@ tests: update: { $set: { x: 22 } } expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + + - - description: 'collection.bulkWrite retries at most maxAttempts times (maxAttempts=5)' + description: 'collection.bulkWrite retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -672,7 +1543,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [insert] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -687,8 +1558,39 @@ tests: document: { _id: 2, x: 22 } expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + + - - description: 'collection.createIndex retries at most maxAttempts times (maxAttempts=5)' + description: 'collection.createIndex retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -696,7 +1598,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [createIndexes] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -710,8 +1612,39 @@ tests: name: "x_11" expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: createIndexes + - commandFailedEvent: + commandName: createIndexes + - commandStartedEvent: + commandName: createIndexes + - commandFailedEvent: + commandName: createIndexes + - commandStartedEvent: + commandName: createIndexes + - commandFailedEvent: + commandName: createIndexes + - commandStartedEvent: + commandName: createIndexes + - commandFailedEvent: + commandName: createIndexes + - commandStartedEvent: + commandName: createIndexes + - commandFailedEvent: + commandName: createIndexes + - commandStartedEvent: + commandName: createIndexes + - commandFailedEvent: + commandName: createIndexes + + - - description: 'collection.dropIndex retries at most maxAttempts times (maxAttempts=5)' + description: 'collection.dropIndex retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -719,7 +1652,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [dropIndexes] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -732,8 +1665,39 @@ tests: name: "x_11" expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + + - - description: 'collection.dropIndexes retries at most maxAttempts times (maxAttempts=5)' + description: 'collection.dropIndexes retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -741,7 +1705,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [dropIndexes] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -751,3 +1715,34 @@ tests: object: *collection name: dropIndexes expectError: true + + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + diff --git a/test/spec/client-backpressure/backpressure-retry-max-attempts.yml.template b/test/spec/client-backpressure/backpressure-retry-max-attempts.yml.template index bf089211fd..555b90425d 100644 --- a/test/spec/client-backpressure/backpressure-retry-max-attempts.yml.template +++ b/test/spec/client-backpressure/backpressure-retry-max-attempts.yml.template @@ -14,6 +14,7 @@ createEntities: client: id: &client client useMultipleMongoses: false + observeEvents: [ 'commandStartedEvent', 'commandSucceededEvent', 'commandFailedEvent' ] - client: @@ -43,6 +44,10 @@ tests: {% for operation in operations %} - description: '{{operation.object}}.{{operation.operation_name}} retries at most maxAttempts=5 times' + {%- if ((operation.operation_name == 'clientBulkWrite')) %} + runOnRequirements: + minServerVersion: '8.0' + {%- endif %} operations: - name: failPoint object: testRunner @@ -50,7 +55,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [{{operation.command_name}}] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -69,4 +74,35 @@ tests: saveResultAsEntity: changeStream {%- endif %} expectError: true + + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: {{operation.command_name}} + - commandFailedEvent: + commandName: {{operation.command_name}} + - commandStartedEvent: + commandName: {{operation.command_name}} + - commandFailedEvent: + commandName: {{operation.command_name}} + - commandStartedEvent: + commandName: {{operation.command_name}} + - commandFailedEvent: + commandName: {{operation.command_name}} + - commandStartedEvent: + commandName: {{operation.command_name}} + - commandFailedEvent: + commandName: {{operation.command_name}} + - commandStartedEvent: + commandName: {{operation.command_name}} + - commandFailedEvent: + commandName: {{operation.command_name}} + - commandStartedEvent: + commandName: {{operation.command_name}} + - commandFailedEvent: + commandName: {{operation.command_name}} + {% endfor -%} From c6bd91cbc09e500b6a157b82ddee645b8ba808c6 Mon Sep 17 00:00:00 2001 From: bailey Date: Tue, 2 Dec 2025 10:12:04 -0700 Subject: [PATCH 09/10] fix tests && re-run lint --- src/index.ts | 1 + src/token_bucket.ts | 3 +++ test/spec/client-backpressure/README.md | 13 +++++++------ .../backpressure-retry-loop.json | 8 +++++--- .../client-backpressure/backpressure-retry-loop.yml | 2 +- .../backpressure-retry-loop.yml.template | 2 +- .../backpressure-retry-max-attempts.json | 8 +++++--- .../backpressure-retry-max-attempts.yml | 2 +- .../backpressure-retry-max-attempts.yml.template | 2 +- 9 files changed, 25 insertions(+), 16 deletions(-) diff --git a/src/index.ts b/src/index.ts index 75acf17c68..26207085e0 100644 --- a/src/index.ts +++ b/src/index.ts @@ -87,6 +87,7 @@ export { MongoWriteConcernError, WriteConcernErrorResult } from './error'; +export { TokenBucket } from './token_bucket'; export { AbstractCursor, // Actual driver classes exported diff --git a/src/token_bucket.ts b/src/token_bucket.ts index fddacdb657..9d542bf719 100644 --- a/src/token_bucket.ts +++ b/src/token_bucket.ts @@ -1,3 +1,6 @@ +/** + * @internal + */ export class TokenBucket { private budget: number; constructor(allowance: number) { diff --git a/test/spec/client-backpressure/README.md b/test/spec/client-backpressure/README.md index 26d664f13e..a4e62b9ec1 100644 --- a/test/spec/client-backpressure/README.md +++ b/test/spec/client-backpressure/README.md @@ -35,7 +35,7 @@ Drivers should test that retries do not occur immediately when a SystemOverloade } ``` - 3. Execute the following command. Expect that the command errors. Measure the duration of the command execution. + 3. Execute the following command. Expect that the command errors. Measure the duration of the command execution. ```javascript const start = performance.now(); @@ -44,16 +44,17 @@ Drivers should test that retries do not occur immediately when a SystemOverloade ).to.be.an.instanceof(MongoServerError); const end = performance.now(); ``` - - 4. Configure the random number generator used for jitter to always return `1`. + + 4. Configure the random number generator used for jitter to always return `1`. + 5. Execute step 3 again. + 6. Compare the two time between the two runs. ```python assertTrue(absolute_value(with_backoff_time - (no_backoff_time + 3.1 seconds)) < 1) ``` - The sum of 5 backoffs is 3.1 seconds. There is a 1-second window to account for potential variance between - the two runs. - + The sum of 5 backoffs is 3.1 seconds. There is a 1-second window to account for potential variance between the two + runs. ## Changelog diff --git a/test/spec/client-backpressure/backpressure-retry-loop.json b/test/spec/client-backpressure/backpressure-retry-loop.json index ae944f73ad..749159d4ae 100644 --- a/test/spec/client-backpressure/backpressure-retry-loop.json +++ b/test/spec/client-backpressure/backpressure-retry-loop.json @@ -381,9 +381,11 @@ }, { "description": "client.clientBulkWrite retries using operation loop", - "runOnRequirements": { - "minServerVersion": "8.0" - }, + "runOnRequirements": [ + { + "minServerVersion": "8.0" + } + ], "operations": [ { "object": "utilCollection", diff --git a/test/spec/client-backpressure/backpressure-retry-loop.yml b/test/spec/client-backpressure/backpressure-retry-loop.yml index c612986233..c069afa6be 100644 --- a/test/spec/client-backpressure/backpressure-retry-loop.yml +++ b/test/spec/client-backpressure/backpressure-retry-loop.yml @@ -224,7 +224,7 @@ tests: - description: 'client.clientBulkWrite retries using operation loop' runOnRequirements: - minServerVersion: '8.0' + - minServerVersion: '8.0' # client bulk write added to server in 8.0 operations: - object: *utilCollection diff --git a/test/spec/client-backpressure/backpressure-retry-loop.yml.template b/test/spec/client-backpressure/backpressure-retry-loop.yml.template index cad83625d3..6101d17178 100644 --- a/test/spec/client-backpressure/backpressure-retry-loop.yml.template +++ b/test/spec/client-backpressure/backpressure-retry-loop.yml.template @@ -58,7 +58,7 @@ tests: description: '{{operation.object}}.{{operation.operation_name}} retries using operation loop' {%- if ((operation.operation_name == 'clientBulkWrite')) %} runOnRequirements: - minServerVersion: '8.0' + - minServerVersion: '8.0' # client bulk write added to server in 8.0 {%- endif %} operations: - diff --git a/test/spec/client-backpressure/backpressure-retry-max-attempts.json b/test/spec/client-backpressure/backpressure-retry-max-attempts.json index 7e9cc67d7a..5cc90248d2 100644 --- a/test/spec/client-backpressure/backpressure-retry-max-attempts.json +++ b/test/spec/client-backpressure/backpressure-retry-max-attempts.json @@ -364,9 +364,11 @@ }, { "description": "client.clientBulkWrite retries at most maxAttempts=5 times", - "runOnRequirements": { - "minServerVersion": "8.0" - }, + "runOnRequirements": [ + { + "minServerVersion": "8.0" + } + ], "operations": [ { "name": "failPoint", diff --git a/test/spec/client-backpressure/backpressure-retry-max-attempts.yml b/test/spec/client-backpressure/backpressure-retry-max-attempts.yml index 9161828e99..9bbbd74a49 100644 --- a/test/spec/client-backpressure/backpressure-retry-max-attempts.yml +++ b/test/spec/client-backpressure/backpressure-retry-max-attempts.yml @@ -203,7 +203,7 @@ tests: - description: 'client.clientBulkWrite retries at most maxAttempts=5 times' runOnRequirements: - minServerVersion: '8.0' + - minServerVersion: '8.0' # client bulk write added to server in 8.0 operations: - name: failPoint object: testRunner diff --git a/test/spec/client-backpressure/backpressure-retry-max-attempts.yml.template b/test/spec/client-backpressure/backpressure-retry-max-attempts.yml.template index 555b90425d..342556a6c3 100644 --- a/test/spec/client-backpressure/backpressure-retry-max-attempts.yml.template +++ b/test/spec/client-backpressure/backpressure-retry-max-attempts.yml.template @@ -46,7 +46,7 @@ tests: description: '{{operation.object}}.{{operation.operation_name}} retries at most maxAttempts=5 times' {%- if ((operation.operation_name == 'clientBulkWrite')) %} runOnRequirements: - minServerVersion: '8.0' + - minServerVersion: '8.0' # client bulk write added to server in 8.0 {%- endif %} operations: - name: failPoint From f30f570e968c4c4f5d0d828240f4a843d81aaca8 Mon Sep 17 00:00:00 2001 From: bailey Date: Wed, 26 Nov 2025 14:19:25 -0700 Subject: [PATCH 10/10] add prose test for handshake construction changes --- src/cmap/connect.ts | 2 + .../mongodb-handshake.prose.test.ts | 44 ++++++++++++++++++- 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/src/cmap/connect.ts b/src/cmap/connect.ts index 62db2f98d4..c8ad9e23b2 100644 --- a/src/cmap/connect.ts +++ b/src/cmap/connect.ts @@ -209,6 +209,7 @@ export interface HandshakeDocument extends Document { compression: string[]; saslSupportedMechs?: string; loadBalanced?: boolean; + backpressure: true; } /** @@ -226,6 +227,7 @@ export async function prepareHandshakeDocument( const handshakeDoc: HandshakeDocument = { [serverApi?.version || options.loadBalanced === true ? 'hello' : LEGACY_HELLO_COMMAND]: 1, + backpressure: true, helloOk: true, client: clientMetadata, compression: compressors diff --git a/test/integration/mongodb-handshake/mongodb-handshake.prose.test.ts b/test/integration/mongodb-handshake/mongodb-handshake.prose.test.ts index a8ac32b99c..d41f46e23e 100644 --- a/test/integration/mongodb-handshake/mongodb-handshake.prose.test.ts +++ b/test/integration/mongodb-handshake/mongodb-handshake.prose.test.ts @@ -1,7 +1,14 @@ import { expect } from 'chai'; import * as sinon from 'sinon'; -import { type ClientMetadata, type DriverInfo, Int32, MongoClient } from '../../../src'; +import { + type ClientMetadata, + type Document, + type DriverInfo, + type HandshakeDocument, + Int32, + MongoClient +} from '../../../src'; import { Connection } from '../../../src/cmap/connection'; import { getFAASEnv, isDriverInfoEqual } from '../../../src/cmap/handshake/client_metadata'; import { LEGACY_HELLO_COMMAND } from '../../../src/constants'; @@ -939,3 +946,38 @@ describe('Client Metadata Update Prose Tests', function () { } }); }); + +// TODO: add prose test descriptions here to align the test with the spec. +describe.only('Backpressure Metadata', function () { + let client: MongoClient; + let spy: sinon.SinonSpy>; + + beforeEach(async function () { + client = this.configuration.newClient(); + spy = sinon.spy(Connection.prototype, 'command'); + await client.connect(); + + // run an operation to force a connection establishment, + // if we're testing noauth load balanced mode. + await client.db('foo').collection('bar').insertOne({ name: 'bumpy' }); + }); + + afterEach(async function () { + sinon.restore(); + await client?.close(); + }); + + it('includes backpressure in the handshake document', function () { + const isHello = (cmd: Document): cmd is HandshakeDocument => + `hello` in cmd || LEGACY_HELLO_COMMAND in cmd; + + const hellos = spy.args.map(([_ns, command, _options]) => command).filter(isHello); + + expect(hellos.length).to.be.greaterThan(0); + + expect( + hellos.every(hello => hello.backpressure === true), + `some handshake documents did not specify backpressure: true` + ); + }); +});