From 588f1f2ff406239b178ff035726e524025ec72c3 Mon Sep 17 00:00:00 2001 From: bailey Date: Tue, 25 Nov 2025 16:00:54 -0700 Subject: [PATCH 01/12] initial commit --- source/index.md | 1 + source/mongodb-handshake/handshake.md | 1 + source/retryable-reads/retryable-reads.md | 13 +++++++++++++ source/retryable-writes/retryable-writes.md | 19 +++++++++++++++---- 4 files changed, 30 insertions(+), 4 deletions(-) diff --git a/source/index.md b/source/index.md index 29b2a51e86..bcff8b0ffd 100644 --- a/source/index.md +++ b/source/index.md @@ -12,6 +12,7 @@ - [CRUD API](crud/crud.md) - [Causal Consistency Specification](causal-consistency/causal-consistency.md) - [Change Streams](change-streams/change-streams.md) +- [Client Backpressure](client-backpressure/client-backpressure.md) - [Client Side Encryption](client-side-encryption/client-side-encryption.md) - [Client Side Operations Timeout](client-side-operations-timeout/client-side-operations-timeout.md) - [Collation](collation/collation.md) diff --git a/source/mongodb-handshake/handshake.md b/source/mongodb-handshake/handshake.md index 24c6eea50d..5a62627326 100644 --- a/source/mongodb-handshake/handshake.md +++ b/source/mongodb-handshake/handshake.md @@ -84,6 +84,7 @@ if stable_api_configured or client_options.load_balanced: else: cmd = {"legacy hello": 1, "helloOk": 1} conn.supports_op_msg = False # Send the initial command via OP_QUERY. +cmd["backpressure"] = True cmd["client"] = client_metadata if client_options.compressors: cmd["compression"] = client_options.compressors diff --git a/source/retryable-reads/retryable-reads.md b/source/retryable-reads/retryable-reads.md index d715f774ef..a7100cbe92 100644 --- a/source/retryable-reads/retryable-reads.md +++ b/source/retryable-reads/retryable-reads.md @@ -15,6 +15,10 @@ This specification will - outline how an API for retryable read operations will be implemented in drivers - define an option to enable retryable reads for an application. +The changes in this specification are related to but distinct from the retryability behaviors defined in the client +backpressure specification, which defines a retryability mechanism for all commands under certain server conditions. +Unless otherwise noted, the changes in this specification refer only to the retryability behaviors summarized above. + ## META The keywords "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and @@ -267,6 +271,13 @@ The following pseudocode for executing retryable read commands has been adapted [the pseudocode for executing retryable write commands](../retryable-writes/retryable-writes.md#executing-retryable-write-commands) and reflects the flow described above. +> [!NOTE] +> The rules above and the pseudocode below only demonstrate the rules for retryable reads as outlined in this +> specification. For simplicity, and to make the retryable reads rules easier to follow, the pseudocode was +> intentionally unmodified. For a pseudocode block that contains both retryable reads logic as defined in this +> specification and backoff retryabilitity as defined in the client backpressure specification, see the pseudocode in +> the [Backpressure Specification](../client-backpressure/client-backpressure.md). + ```typescript /** * Checks if a connection supports retryable reads. @@ -547,6 +558,8 @@ any customers experiencing degraded performance can simply disable `retryableRea ## Changelog +- xxxx-xx-xx: Clarify handling of deprioritized servers in pseudocode. + - 2024-04-30: Migrated from reStructuredText to Markdown. - 2023-12-05: Add that any server information associated with retryable exceptions MUST reflect the originating server, diff --git a/source/retryable-writes/retryable-writes.md b/source/retryable-writes/retryable-writes.md index 609de18b92..fa613a2908 100644 --- a/source/retryable-writes/retryable-writes.md +++ b/source/retryable-writes/retryable-writes.md @@ -19,6 +19,10 @@ specification will outline how an API for retryable write operations will be imp will define an option to enable retryable writes for an application and describe how a transaction ID will be provided to write commands executed therein. +The changes in this specification are related to but distinct from the retryability behaviors defined in the client +backpressure specification, which defines a retryability mechanism for all commands under certain server conditions. +Unless otherwise noted, the changes in this specification refer only to the retryability behaviors summarized above. + ## META The keywords "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and @@ -43,10 +47,10 @@ specification. This object is always associated with a server session; however, that creating a ClientSession will not always entail creation of a new server session. The name of this object MAY vary across drivers. -**Retryable Error** +**Retryable Write Error** An error is considered retryable if it has a RetryableWriteError label in its top-level "errorLabels" field. See -[Determining Retryable Errors](#determining-retryable-errors) for more information. +[Determining Retryable Write Errors](#determining-retryable-errors) for more information. Additional terms may be defined in the [Driver Session](../sessions/driver-sessions.md) specification. @@ -102,7 +106,7 @@ In a sharded cluster, it is possible that mongos may appear to support retryable cluster do not (e.g. replica set shard is configured with feature compatibility version 3.4, a standalone is added as a new shard). In these rare cases, a write command that fans out to a shard that does not support retryable writes may partially fail and an error may be reported in the write result from mongos (e.g. `writeErrors` array in the bulk write -result). This does not constitute a retryable error. Drivers MUST relay such errors to the user. +result). This does not constitute a retryable write error. Drivers MUST relay such errors to the user. #### Supported Write Operations @@ -162,7 +166,7 @@ occurs during a write command within a transaction (excepting `commitTransation` ### Implementing Retryable Writes -#### Determining Retryable Errors +#### Determining Retryable Write Errors When connected to a MongoDB instance that supports retryable writes (versions 3.6+), the driver MUST treat all errors with the RetryableWriteError label as retryable. This error label can be found in the top-level "errorLabels" field of @@ -333,6 +337,13 @@ errors are labeled "NoWritesPerformed", then the first error should be raised. If a driver associates server information (e.g. the server address or description) with an error, the driver MUST ensure that the reported server information corresponds to the server that originated the error. +> [!NOTE] +> The rules above and the pseudocode below only demonstrate the rules for retryable writes as outlined in this +> specification. For simplicity, and to make the retryable writes rules easier to follow, the pseudocode was +> intentionally unmodified. For a pseudocode block that contains both retryable writes logic as defined in this +> specification and backoff retryabilitity as defined in the client backpressure specification, see the pseudocode in +> the [Backpressure Specification](../client-backpressure/client-backpressure.md). + The above rules are implemented in the following pseudo-code: ```typescript From e467f5bc0409b550ba3f392e6b163f97337ba901 Mon Sep 17 00:00:00 2001 From: bailey Date: Wed, 26 Nov 2025 10:15:49 -0700 Subject: [PATCH 02/12] new files --- .../client-backpressure.md | 424 ++++++++++++++++++ source/client-backpressure/tests/README.md | 15 + 2 files changed, 439 insertions(+) create mode 100644 source/client-backpressure/client-backpressure.md create mode 100644 source/client-backpressure/tests/README.md diff --git a/source/client-backpressure/client-backpressure.md b/source/client-backpressure/client-backpressure.md new file mode 100644 index 0000000000..88dd94807e --- /dev/null +++ b/source/client-backpressure/client-backpressure.md @@ -0,0 +1,424 @@ +# Client Backpressure + +- Status: Accepted +- Minimum Server Version: N/A + +______________________________________________________________________ + +## Abstract + +This specification adds the ability for drivers to automatically retry requests that fail due to server overload errors +while applying backpressure to avoid further overloading the server. + +## META + +The keywords "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and +"OPTIONAL" in this document are to be interpreted as described in [RFC 2119](https://www.ietf.org/rfc/rfc2119.txt). + +## Specification + +### Terms + +#### Ingress Connection Rate Limiter + +A token-bucket based system introduced in MongoDB 8.2 to admit, reject or queue connection requests. It aims to prevent +connection spikes from overloading the system. + +#### Ingress Request Rate Limiter + +A token bucket based system introduced in MongoDB 8.2 to admit an operation or reject it with a System Overload Error at +the front door of a mongod/s. It aims to prevent operations spikes from overloading the system. + +#### MongoTune + +Mongotune is a policy engine outside the server (mongod or mongos) which monitors a set of metrics (MongoDB or system +host) to dynamically configure MongoDB settings. MongoTune is deployed to Atlas clusters and will dynamically configure +the connection and request rate limiters to prevent and mitigate overloading the system. + +#### RetryableError label + +An error is considered retryable if it includes the "RetryableError" label. This error label indicates that an operation +is safely retryable regardless of the type of operation, its metadata, or any of its arguments. + +#### SystemOverloadedError label + +An error is considered overloaded if it includes the "SystemOverloadError" label. This error label indicates that the +server is overloaded. If this error label is present, drivers will backoff before attempting a retry. + +#### Overload Errors + +An overload error is any command or network error that occurs due to a server overload. For example, when a request +exceeds the ingress request rate limit: + +```js +{ + 'ok': 0.0, + 'errmsg': "Rate limiter 'ingressRequestRateLimiter' rate exceeded", + 'code': 462, + 'codeName': 'IngressRequestRateLimitExceeded', + 'errorLabels': ['SystemOverloadedError', 'RetryableError'], +} +``` + +When a new connection attempt exceeds the ingress connection rate limit, the server closes the TCP connection before TLS +handshake is complete. Drivers will observe this as a network error (e.g. "connection reset by peer" or "connection +closed"). + +When a new connection attempt is queued by the server for so long that the driver-side timeout expires, drivers will +observe this as a network timeout error. + +### Requirements for Client Backpressure + +#### Overload retry policy + +This specification expands the driver's retry ability to all commands, including those not currently considered +retryable such as updateMany, create collection, getMore, and generic runCommand. The new command execution method obeys +the following rules: + +1. If the command succeeds on the first attempt, drivers MUST deposit `RETRY_TOKEN_RETURN_RATE` tokens. + - The value is 0.1 and non-configurable. +2. If the command succeeds on a retry attempt, drivers MUST deposit `RETRY_TOKEN_RETURN_RATE`+1 tokens. +3. If a retry attempt fails with an error that does not include `SystemOverloadedError` label, drivers MUST deposit 1 + token. +4. A retry attempt will only be permitted if the error includes the `RetryableError` label, we have not reached + `MAX_ATTEMPTS`, the CSOT deadline has not expired, and a token can be acquired from the token bucket. + - The value of `MAX_ATTEMPTS` is 5 and non-configurable. + - This intentionally changes the behavior of CSOT which otherwise would retry an unlimited number of times within the + timeout to avoid retry storms. +5. If the previous error includes the `SystemOverloadedError` label, the client MUST apply exponential backoff according + to according to the following formula: `delayMS = j * min(maxBackoff, baseBackoff * 2^i)` + - `i` is the retry attempt (starting with 0 for the first retry). + - `j` is a random jitter value between 0 and 1. + - `baseBackoff` is constant 100ms. + - `maxBackoff` is 10000ms. + - This results in delays of 100ms, 200ms, 400ms, 800ms, and 1600ms before accounting for jitter. +6. If the previous error contained the `SystemOverloadedError` error label, the node will be added to the set of + deprioritized servers. + +#### Pseudocode + +The following pseudocode describes the overload retry policy: + +```python +BASE_BACKOFF = 0.1 +MAX_BACKOFF = 10 +RETRY_TOKEN_RETURN_RATE = 0.1 + +def execute_command_retryable(command, ...): + deprioritized_servers = [] + attempt = 0 + while True: + try: + server = select_server(deprioritized_servers) + connection = server.getConnection() + res = execute_command(connection, command) + # Return tokens to the bucket on success. + tokens = RETRY_TOKEN_RETURN_RATE + if attempt > 0: + tokens += 1 + token_bucket.deposit(tokens) + return res + except PyMongoError as exc: + backoff = 0 + attempt += 1 + + if attempt > MAX_ATTEMPTS: + raise + + # Raise if the error is non retryable. + is_retryable = exc.has_error_label("RetryableError") or is_retryable_write_error() or is_retryable_read_error() + if not is_retryable: + raise error + if exc.has_error_label("SystemOverloadedError"): + jitter = random.random() # Random float between [0.0, 1.0). + backoff = jitter * min(BASE_BACKOFF * (2 ** attempt), MAX_BACKOFF) + + # If the delay exceeds the deadline, bail early before consuming a token. + if _csot.get_timeout(): + if time.monotonic() + backoff > _csot.get_deadline(): + raise + + if not token_bucket.consume(1): + raise + + if backoff: + time.sleep(backoff) + deprioritized_servers.append(server) + continue +``` + +Some drivers might not have retryability implementations that allow easy separation of the existing retryable +reads/writes mechanisms from the exponential backoff and jitter retry algorithm. An example pseudocode is defined below +that demonstrates a combined retryable reads/writes implementation with the corresponding backpressure changes (adapted +from the Node driver's implementation): + +```typescript +async function tryOperation>( + operation: T, + { topology, timeoutContext, session, readPreference }: RetryOptions +): Promise { + const serverSelector = getServerSelectorForReadPreference(operation, readPreference); + + let server = await topology.selectServer(selector, { + session, + }); + + const hasReadAspect = operation.hasAspect(Aspect.READ_OPERATION); + const hasWriteAspect = operation.hasAspect(Aspect.WRITE_OPERATION); + const inTransaction = session?.inTransaction() ?? false; + + const willRetryRead = topology.s.options.retryReads && !inTransaction && operation.canRetryRead; + + const willRetryWrite = + topology.s.options.retryWrites && + !inTransaction && + supportsRetryableWrites(server) && + operation.canRetryWrite; + + const willRetry = + operation.hasAspect(Aspect.RETRYABLE) && + session != null && + ((hasReadAspect && willRetryRead) || (hasWriteAspect && willRetryWrite)); + + if (hasWriteAspect && willRetryWrite && session != null) { + operation.options.willRetryWrite = true; + session.incrementTransactionNumber(); + } + + // The maximum number of retry attempts using regular retryable reads/writes logic (not including + // SystemOverLoad error retries). + const maxNonOverloadRetryAttempts = willRetry + ? timeoutMS != null + ? Infinity + : 2 + : 1; + + let previousOperationError: MongoError | undefined; + let previousServer: ServerDescription | undefined; + + let nonOverloadRetryAttempt = 0; + let systemOverloadRetryAttempt = 0; + + const maxSystemOverloadRetryAttempts = 5; + const backoffDelayProvider = exponentialBackoffDelayProvider( + 10_000, // MAX_BACKOFF + 100, // base backoff + 2 // backoff rate + ); + + while (true) { + if (previousOperationError) { + if (previousOperationError.hasErrorLabel("SystemOverloadError")) { + systemOverloadRetryAttempt += 1; + + if ( + // if the SystemOverloadError is not retryable, throw. + !previousOperationError.hasErrorLabel("RetryableError") || + !( + // if retryable writes or reads are not configured, throw. + ( + (hasReadAspect && topology.s.options.retryReads) || + (hasWriteAspect && topology.s.options.retryWrites) + ) + ) + ) { + throw previousOperationError; + } + + // if we have exhausted overload retry attempts, throw. + if (systemOverloadRetryAttempt > maxSystemOverloadRetryAttempts) { + throw previousOperationError; + } + + const { value: delayMS } = backoffDelayProvider.next(); + + // if the delay would exhaust the CSOT timeout, short-circuit. + if (timeoutContext.csotEnabled() && delayMS > timeoutContext.remainingTimeMS) { + throw previousError; + } + + await setTimeout(delayMS); + + // attempt to consume a retry token, throw if we don't have budget. + if (!topology.tokenBucket.consume(RETRY_COST)) { + throw previousOperationError; + } + + server = await topology.selectServer(selector, { session }); + } else { + nonOverloadRetryAttempt++; + // we have no more retry attempts, throw. + if (nonOverloadRetryAttempt > maxNonOverloadRetryAttempts) { + throw previousOperationError; + } + + // Handle MMAPv1 not supporting retryable writes. + if (hasWriteAspect && previousOperationError.code === MMAPv1_RETRY_WRITES_ERROR_CODE) { + throw new MongoServerError({ + message: MMAPv1_RETRY_WRITES_ERROR_MESSAGE, + errmsg: MMAPv1_RETRY_WRITES_ERROR_MESSAGE, + originalError: previousOperationError + }); + } + + // handle non-retryable errors + if ( + (hasWriteAspect && !isRetryableWriteError(previousOperationError)) || + (hasReadAspect && !isRetryableReadError(previousOperationError)) + ) { + throw previousOperationError; + } + + server = await topology.selectServer(selector, { session }); + + // handle rare downgrade scenarios where some nodes don't support + // retryable writes but others do. + if (hasWriteAspect && !supportsRetryableWrites(server)) { + throw new MongoUnexpectedServerResponseError( + 'Selected server does not support retryable writes' + ); + } + } + } + + try { + try { + const result = await server.command(operation, timeoutContext); + const isRetry = nonOverloadRetryAttempt > 0 || systemOverloadRetryAttempt > 0; + topology.tokenBucket.deposit( + isRetry + ? // on successful retry, deposit the retry cost + the refresh rate. + TOKEN_REFRESH_RATE + RETRY_COST + : // otherwise, just deposit the refresh rate. + TOKEN_REFRESH_RATE + ); + return operation.handleOk(result); + } catch (error) { + return operation.handleError(error); + } + } catch (operationError) { + if (!operationError.hasErrorLabel("SystemOverloadError")) { + // if an operation fails with an error that does not contain the SystemOverloadError, deposit 1 token. + topology.tokenBucket.deposit(RETRY_COST); + } + + if ( + previousOperationError != null && + operationError.hasErrorLabel("NoWritesPerformed") + ) { + throw previousOperationError; + } + previousServer = server.description; + previousOperationError = operationError; + } + } +} +``` + +### Token Bucket + +The overload retry policy introduces a per-client token bucket to limit retry attempts. Although the server rejects +excess operations as quickly as possible, doing so costs CPU and creates extra contention on the connection pool which +can eventually negatively affect goodput. To reduce this risk, the token bucket will limit retry attempts during a +prolonged overload. + +The token bucket capacity is set to 1000 for consistency with the server. + +#### Pseudocode + +The token bucket is implemented via a thread safe counter. For languages without atomics, this can be implemented via a +lock, for example: + +```python +DEFAULT_RETRY_TOKEN_CAPACITY = 1000 +class TokenBucket: + """A token bucket implementation for rate limiting.""" + def __init__( + self, + capacity: float = DEFAULT_RETRY_TOKEN_CAPACITY, + ): + self.lock = Lock() + self.capacity = capacity + self.tokens = capacity + + def consume(self, n: float) -> bool: + """Consume n tokens from the bucket if available.""" + with self.lock: + if self.tokens >= n: + self.tokens -= n + return True + return False + + def deposit(self, n: float) -> None: + """Deposit n tokens back into the bucket.""" + with self.lock: + self.tokens = min(self.capacity, self.tokens + n) +``` + +#### Handshake changes + +Drivers conforming to this spec MUST add `“backpressure”: True` to the connection handshake. This flag allows the server +to identify clients which do and do not support backpressure. Currently, this flag is unused but in the future the +server may offer different rate limiting behavior for clients that do not support backpressure. + +##### Implementation notes + +On some platforms sleep() can have a very low precision, meaning an attempt to sleep for 50ms may actually sleep for a +much larger time frame. Drivers are not required to work around this limitation. + +### Logging Retry Attempts + +[As with retryable writes](../retryable-writes/retryable-writes.md#logging-retry-attempts), drivers MAY choose to log +retry attempts for load shed operations. This specification does not define a format for such log messages. + +### Command Monitoring + +[As with retryable writes](../retryable-writes/retryable-writes.md#command-monitoring), in accordance with the +[Command Logging and Monitoring](../command-logging-and-monitoring/command-logging-and-monitoring.md) specification, +drivers MUST guarantee that each `CommandStartedEvent` has either a correlating `CommandSucceededEvent` or +`CommandFailedEvent` and that every "command started" log message has either a correlating "command succeeded" log +message or "command failed" log message. If the first attempt of a retryable operation encounters a retryable error, +drivers MUST fire a `CommandFailedEvent` and emit a "command failed" log message for the retryable error and fire a +separate `CommandStartedEvent` and emit a separate "command started" log message when executing the subsequent retry +attempt. Note that the second `CommandStartedEvent` and "command started" log message may have a different +`connectionId`, since a server is reselected for a retry attempt. + +### Documentation + +1. Drivers MUST document that all operations support retries on server overload. +2. Driver release notes MUST make it clear to users that they may need to adjust custom retry logic to prevent an + application from inadvertently retrying for too long (see [Backwards Compatibility](#backwards-compatibility) for + details). + +## Test Plan + +See the [README](./tests/README.md) for tests. + +## Motivation for Change + +New load shedding mechanisms are being introduced to the server that improve its ability to remain available under +extreme load, however clients do not know how to handle the errors returned when one of its requests has been rejected. +As a result, such overload errors would currently either be propagated back to applications, increasing +externally-visible command failure rates, or be retried immediately, increasing the load on already overburdened +servers. To minimize these effects, this specification enables clients to retry requests that have been load shed in a +way that does not overburden already overloaded servers. This retry behavior allows for more aggressive and effective +load shedding policies to be deployed in the future. This will also help unify the currently-divergent retry behavior +between drivers and the server (mongos). + +## Reference Implementation + +The Node and Python drivers will provide the reference implementations. See +[NODE-7142](https://jira.mongodb.org/browse/NODE-7142) and [PYTHON-5528](https://jira.mongodb.org/browse/PYTHON-5528). + +## Future work + +1. [DRIVERS-3333](https://jira.mongodb.org/browse/DRIVERS-3333) Add a backoff state into the connection pool. +2. [DRIVERS-3241](https://jira.mongodb.org/browse/DRIVERS-3241) Add diagnostic metadata to retried commands. + +## Q&A + +TODO + +## Changelog + +- 2025-XX-XX: Initial version. diff --git a/source/client-backpressure/tests/README.md b/source/client-backpressure/tests/README.md new file mode 100644 index 0000000000..7a70e4ad76 --- /dev/null +++ b/source/client-backpressure/tests/README.md @@ -0,0 +1,15 @@ +# Client Backpressure Tests + +______________________________________________________________________ + +## Introduction + +The YAML and JSON files in this directory are platform-independent tests meant to exercise a driver's implementation of +retryable reads. These tests utilize the [Unified Test Format](../../unified-test-format/unified-test-format.md). + +Several prose tests, which are not easily expressed in YAML, are also presented in this file. Those tests will need to +be manually implemented by each driver. + +## Changelog + +- 2025-XX-XX: Initial version. From d55fdb9ebbcecc5f9f15d16b8d9d5151495b3713 Mon Sep 17 00:00:00 2001 From: bailey Date: Wed, 26 Nov 2025 14:25:13 -0700 Subject: [PATCH 03/12] add tests for handshake changes --- source/mongodb-handshake/tests/README.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/source/mongodb-handshake/tests/README.md b/source/mongodb-handshake/tests/README.md index d88a49fa92..0ba58b713a 100644 --- a/source/mongodb-handshake/tests/README.md +++ b/source/mongodb-handshake/tests/README.md @@ -486,3 +486,17 @@ Before each test case, perform the setup. 7. Store the response as `updatedClientMetadata`. 8. Assert that `initialClientMetadata` is identical to `updatedClientMetadata`. + +### Test 9: Handshake documents include `backpressure: true` + +These tests require a mechanism for observing handshake documents sent to the server. + +1. Create a `MongoClient` that is configured to record all handshake documents sent to the server as a part of + connection establishment. + +2. Send a `ping` command to the server and verify that the command succeeds. This ensure that a connection is + established on all topologies. + +3. Assert that for every handshake document intercepted: + + 1. the document has a field `backpressure` whose value is `true`. From 8e74b418404a573d6d622122e031b088871d169b Mon Sep 17 00:00:00 2001 From: bailey Date: Mon, 1 Dec 2025 15:59:14 -0700 Subject: [PATCH 04/12] add generated tests --- .../tests/backpressure-retry-loop.json | 3403 +++++++++++++++++ .../tests/backpressure-retry-loop.yml | 1860 +++++++++ .../backpressure-retry-loop.yml.template | 128 + .../backpressure-retry-max-attempts.json | 1262 ++++++ .../tests/backpressure-retry-max-attempts.yml | 753 ++++ ...ckpressure-retry-max-attempts.yml.template | 72 + ...enerate-backpressure-retryability-tests.py | 125 + 7 files changed, 7603 insertions(+) create mode 100644 source/client-backpressure/tests/backpressure-retry-loop.json create mode 100644 source/client-backpressure/tests/backpressure-retry-loop.yml create mode 100644 source/client-backpressure/tests/backpressure-retry-loop.yml.template create mode 100644 source/client-backpressure/tests/backpressure-retry-max-attempts.json create mode 100644 source/client-backpressure/tests/backpressure-retry-max-attempts.yml create mode 100644 source/client-backpressure/tests/backpressure-retry-max-attempts.yml.template create mode 100644 source/etc/generate-backpressure-retryability-tests.py diff --git a/source/client-backpressure/tests/backpressure-retry-loop.json b/source/client-backpressure/tests/backpressure-retry-loop.json new file mode 100644 index 0000000000..21fc802344 --- /dev/null +++ b/source/client-backpressure/tests/backpressure-retry-loop.json @@ -0,0 +1,3403 @@ +{ + "description": "tests that operations respect overload backoff retry loop", + "schemaVersion": "1.0", + "runOnRequirements": [ + { + "minServerVersion": "4.4", + "topologies": [ + "replicaset", + "sharded", + "load-balanced" + ] + } + ], + "createEntities": [ + { + "client": { + "id": "client", + "useMultipleMongoses": false, + "observeEvents": [ + "commandStartedEvent", + "commandSucceededEvent", + "commandFailedEvent" + ] + } + }, + { + "client": { + "id": "failPointClient", + "useMultipleMongoses": false + } + }, + { + "database": { + "id": "utilDb", + "client": "failPointClient", + "databaseName": "retryable-writes-tests" + } + }, + { + "collection": { + "id": "utilCollection", + "database": "utilDb", + "collectionName": "coll" + } + }, + { + "database": { + "id": "database", + "client": "client", + "databaseName": "retryable-writes-tests" + } + }, + { + "collection": { + "id": "collection", + "database": "database", + "collectionName": "coll" + } + } + ], + "initialData": [ + { + "collectionName": "coll", + "databaseName": "retryable-writes-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + } + ] + } + ], + "tests": [ + { + "description": "client.listDatabases retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "listDatabases" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "client", + "name": "listDatabases", + "arguments": { + "filter": {} + }, + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandSucceededEvent": { + "commandName": "listDatabases" + } + } + ] + } + ] + }, + { + "description": "client.listDatabaseNames retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "listDatabases" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "client", + "name": "listDatabaseNames", + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandSucceededEvent": { + "commandName": "listDatabases" + } + } + ] + } + ] + }, + { + "description": "client.createChangeStream retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "client", + "name": "createChangeStream", + "arguments": { + "pipeline": [] + }, + "saveResultAsEntity": "changeStream", + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandSucceededEvent": { + "commandName": "aggregate" + } + } + ] + } + ] + }, + { + "description": "client.clientBulkWrite retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "bulkWrite" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "client", + "name": "clientBulkWrite", + "arguments": { + "models": [ + { + "insertOne": { + "namespace": "retryable-writes-tests.coll", + "document": { + "_id": 8, + "x": 88 + } + } + } + ] + }, + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandFailedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandStartedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandFailedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandStartedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandFailedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandStartedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandSucceededEvent": { + "commandName": "bulkWrite" + } + } + ] + } + ] + }, + { + "description": "database.aggregate retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "database", + "name": "aggregate", + "arguments": { + "pipeline": [ + { + "$listLocalSessions": {} + }, + { + "$limit": 1 + } + ] + }, + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandSucceededEvent": { + "commandName": "aggregate" + } + } + ] + } + ] + }, + { + "description": "database.listCollections retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "listCollections" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "database", + "name": "listCollections", + "arguments": { + "filter": {} + }, + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandSucceededEvent": { + "commandName": "listCollections" + } + } + ] + } + ] + }, + { + "description": "database.listCollectionNames retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "listCollections" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "database", + "name": "listCollectionNames", + "arguments": { + "filter": {} + }, + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandSucceededEvent": { + "commandName": "listCollections" + } + } + ] + } + ] + }, + { + "description": "database.runCommand retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "ping" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "database", + "name": "runCommand", + "arguments": { + "command": { + "ping": 1 + }, + "commandName": "ping" + }, + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "ping" + } + }, + { + "commandFailedEvent": { + "commandName": "ping" + } + }, + { + "commandStartedEvent": { + "commandName": "ping" + } + }, + { + "commandFailedEvent": { + "commandName": "ping" + } + }, + { + "commandStartedEvent": { + "commandName": "ping" + } + }, + { + "commandFailedEvent": { + "commandName": "ping" + } + }, + { + "commandStartedEvent": { + "commandName": "ping" + } + }, + { + "commandSucceededEvent": { + "commandName": "ping" + } + } + ] + } + ] + }, + { + "description": "database.createChangeStream retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "database", + "name": "createChangeStream", + "arguments": { + "pipeline": [] + }, + "saveResultAsEntity": "changeStream", + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandSucceededEvent": { + "commandName": "aggregate" + } + } + ] + } + ] + }, + { + "description": "collection.aggregate retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "aggregate", + "arguments": { + "pipeline": [] + }, + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandSucceededEvent": { + "commandName": "aggregate" + } + } + ] + } + ] + }, + { + "description": "collection.countDocuments retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "countDocuments", + "arguments": { + "filter": {} + }, + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandSucceededEvent": { + "commandName": "aggregate" + } + } + ] + } + ] + }, + { + "description": "collection.estimatedDocumentCount retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "count" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "estimatedDocumentCount", + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "count" + } + }, + { + "commandFailedEvent": { + "commandName": "count" + } + }, + { + "commandStartedEvent": { + "commandName": "count" + } + }, + { + "commandFailedEvent": { + "commandName": "count" + } + }, + { + "commandStartedEvent": { + "commandName": "count" + } + }, + { + "commandFailedEvent": { + "commandName": "count" + } + }, + { + "commandStartedEvent": { + "commandName": "count" + } + }, + { + "commandSucceededEvent": { + "commandName": "count" + } + } + ] + } + ] + }, + { + "description": "collection.distinct retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "distinct" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "distinct", + "arguments": { + "fieldName": "x", + "filter": {} + }, + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "distinct" + } + }, + { + "commandFailedEvent": { + "commandName": "distinct" + } + }, + { + "commandStartedEvent": { + "commandName": "distinct" + } + }, + { + "commandFailedEvent": { + "commandName": "distinct" + } + }, + { + "commandStartedEvent": { + "commandName": "distinct" + } + }, + { + "commandFailedEvent": { + "commandName": "distinct" + } + }, + { + "commandStartedEvent": { + "commandName": "distinct" + } + }, + { + "commandSucceededEvent": { + "commandName": "distinct" + } + } + ] + } + ] + }, + { + "description": "collection.find retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "find" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "find", + "arguments": { + "filter": {} + }, + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandSucceededEvent": { + "commandName": "find" + } + } + ] + } + ] + }, + { + "description": "collection.findOne retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "find" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "findOne", + "arguments": { + "filter": {} + }, + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandSucceededEvent": { + "commandName": "find" + } + } + ] + } + ] + }, + { + "description": "collection.listIndexes retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "listIndexes" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "listIndexes", + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandSucceededEvent": { + "commandName": "listIndexes" + } + } + ] + } + ] + }, + { + "description": "collection.listIndexNames retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "listIndexes" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "listIndexNames", + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandSucceededEvent": { + "commandName": "listIndexes" + } + } + ] + } + ] + }, + { + "description": "collection.createChangeStream retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "createChangeStream", + "arguments": { + "pipeline": [] + }, + "saveResultAsEntity": "changeStream", + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandSucceededEvent": { + "commandName": "aggregate" + } + } + ] + } + ] + }, + { + "description": "collection.insertOne retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "insert" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "insertOne", + "arguments": { + "document": { + "_id": 2, + "x": 22 + } + }, + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandSucceededEvent": { + "commandName": "insert" + } + } + ] + } + ] + }, + { + "description": "collection.insertMany retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "insert" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "insertMany", + "arguments": { + "documents": [ + { + "_id": 2, + "x": 22 + } + ] + }, + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandSucceededEvent": { + "commandName": "insert" + } + } + ] + } + ] + }, + { + "description": "collection.deleteOne retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "delete" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "deleteOne", + "arguments": { + "filter": {} + }, + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandSucceededEvent": { + "commandName": "delete" + } + } + ] + } + ] + }, + { + "description": "collection.deleteMany retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "delete" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "deleteMany", + "arguments": { + "filter": {} + }, + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandSucceededEvent": { + "commandName": "delete" + } + } + ] + } + ] + }, + { + "description": "collection.replaceOne retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "update" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "replaceOne", + "arguments": { + "filter": {}, + "replacement": { + "x": 22 + } + }, + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandSucceededEvent": { + "commandName": "update" + } + } + ] + } + ] + }, + { + "description": "collection.updateOne retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "update" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "updateOne", + "arguments": { + "filter": {}, + "update": { + "$set": { + "x": 22 + } + } + }, + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandSucceededEvent": { + "commandName": "update" + } + } + ] + } + ] + }, + { + "description": "collection.updateMany retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "update" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "updateMany", + "arguments": { + "filter": {}, + "update": { + "$set": { + "x": 22 + } + } + }, + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandSucceededEvent": { + "commandName": "update" + } + } + ] + } + ] + }, + { + "description": "collection.findOneAndDelete retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "findAndModify" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "findOneAndDelete", + "arguments": { + "filter": {} + }, + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandSucceededEvent": { + "commandName": "findAndModify" + } + } + ] + } + ] + }, + { + "description": "collection.findOneAndReplace retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "findAndModify" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "findOneAndReplace", + "arguments": { + "filter": {}, + "replacement": { + "x": 22 + } + }, + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandSucceededEvent": { + "commandName": "findAndModify" + } + } + ] + } + ] + }, + { + "description": "collection.findOneAndUpdate retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "findAndModify" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "findOneAndUpdate", + "arguments": { + "filter": {}, + "update": { + "$set": { + "x": 22 + } + } + }, + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandSucceededEvent": { + "commandName": "findAndModify" + } + } + ] + } + ] + }, + { + "description": "collection.bulkWrite retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "insert" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "bulkWrite", + "arguments": { + "requests": [ + { + "insertOne": { + "document": { + "_id": 2, + "x": 22 + } + } + } + ] + }, + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandSucceededEvent": { + "commandName": "insert" + } + } + ] + } + ] + }, + { + "description": "collection.createIndex retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "createIndexes" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "createIndex", + "arguments": { + "keys": { + "x": 11 + }, + "name": "x_11" + }, + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandSucceededEvent": { + "commandName": "createIndexes" + } + } + ] + } + ] + }, + { + "description": "collection.dropIndex retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "object": "utilCollection", + "name": "createIndex", + "arguments": { + "keys": { + "x": 11 + }, + "name": "x_11" + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "dropIndexes" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "dropIndex", + "arguments": { + "name": "x_11" + }, + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandSucceededEvent": { + "commandName": "dropIndexes" + } + } + ] + } + ] + }, + { + "description": "collection.dropIndexes retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "dropIndexes" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "dropIndexes", + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandSucceededEvent": { + "commandName": "dropIndexes" + } + } + ] + } + ] + } + ] +} diff --git a/source/client-backpressure/tests/backpressure-retry-loop.yml b/source/client-backpressure/tests/backpressure-retry-loop.yml new file mode 100644 index 0000000000..ec2fa25b8f --- /dev/null +++ b/source/client-backpressure/tests/backpressure-retry-loop.yml @@ -0,0 +1,1860 @@ +# Tests in this file are generated from backpressure-retry-loop.yml.template. + +description: tests that operations respect overload backoff retry loop + +schemaVersion: '1.0' + +runOnRequirements: + - + minServerVersion: '4.4' # failCommand + topologies: [replicaset, sharded, load-balanced] + +createEntities: + - + client: + id: &client client + useMultipleMongoses: false + observeEvents: [ 'commandStartedEvent', 'commandSucceededEvent', 'commandFailedEvent' ] + + - + client: + id: &failPointClient failPointClient + useMultipleMongoses: false + + - + database: + id: &utilDb utilDb + client: *failPointClient + databaseName: &database_name retryable-writes-tests + + - + collection: + id: &utilCollection utilCollection + database: *utilDb + collectionName: &collection_name coll + + - + database: + id: &database database + client: *client + databaseName: &database_name retryable-writes-tests + - + collection: + id: &collection collection + database: *database + collectionName: &collection_name coll + +initialData: + - + collectionName: *collection_name + databaseName: *database_name + documents: + - { _id: 1, x: 11 } + - { _id: 2, x: 22 } + +tests: + + - + description: 'client.listDatabases retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [listDatabases] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *client + name: listDatabases + arguments: + filter: {} + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandSucceededEvent: + commandName: listDatabases + + - + description: 'client.listDatabaseNames retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [listDatabases] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *client + name: listDatabaseNames + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandSucceededEvent: + commandName: listDatabases + + - + description: 'client.createChangeStream retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [aggregate] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *client + name: createChangeStream + arguments: + pipeline: [] + saveResultAsEntity: changeStream + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandSucceededEvent: + commandName: aggregate + + - + description: 'client.clientBulkWrite retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [bulkWrite] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *client + name: clientBulkWrite + arguments: + models: + - insertOne: + namespace: retryable-writes-tests.coll + document: { _id: 8, x: 88 } + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: bulkWrite + - commandFailedEvent: + commandName: bulkWrite + - commandStartedEvent: + commandName: bulkWrite + - commandFailedEvent: + commandName: bulkWrite + - commandStartedEvent: + commandName: bulkWrite + - commandFailedEvent: + commandName: bulkWrite + - commandStartedEvent: + commandName: bulkWrite + - commandSucceededEvent: + commandName: bulkWrite + + - + description: 'database.aggregate retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [aggregate] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *database + name: aggregate + arguments: + pipeline: [ { $listLocalSessions: {} }, { $limit: 1 } ] + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandSucceededEvent: + commandName: aggregate + + - + description: 'database.listCollections retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [listCollections] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *database + name: listCollections + arguments: + filter: {} + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandSucceededEvent: + commandName: listCollections + + - + description: 'database.listCollectionNames retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [listCollections] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *database + name: listCollectionNames + arguments: + filter: {} + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandSucceededEvent: + commandName: listCollections + + - + description: 'database.runCommand retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [ping] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *database + name: runCommand + arguments: + command: { ping: 1 } + commandName: ping + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: ping + - commandFailedEvent: + commandName: ping + - commandStartedEvent: + commandName: ping + - commandFailedEvent: + commandName: ping + - commandStartedEvent: + commandName: ping + - commandFailedEvent: + commandName: ping + - commandStartedEvent: + commandName: ping + - commandSucceededEvent: + commandName: ping + + - + description: 'database.createChangeStream retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [aggregate] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *database + name: createChangeStream + arguments: + pipeline: [] + saveResultAsEntity: changeStream + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandSucceededEvent: + commandName: aggregate + + - + description: 'collection.aggregate retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [aggregate] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: aggregate + arguments: + pipeline: [] + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandSucceededEvent: + commandName: aggregate + + - + description: 'collection.countDocuments retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [aggregate] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: countDocuments + arguments: + filter: {} + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandSucceededEvent: + commandName: aggregate + + - + description: 'collection.estimatedDocumentCount retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [count] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: estimatedDocumentCount + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: count + - commandFailedEvent: + commandName: count + - commandStartedEvent: + commandName: count + - commandFailedEvent: + commandName: count + - commandStartedEvent: + commandName: count + - commandFailedEvent: + commandName: count + - commandStartedEvent: + commandName: count + - commandSucceededEvent: + commandName: count + + - + description: 'collection.distinct retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [distinct] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: distinct + arguments: + fieldName: x + filter: {} + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: distinct + - commandFailedEvent: + commandName: distinct + - commandStartedEvent: + commandName: distinct + - commandFailedEvent: + commandName: distinct + - commandStartedEvent: + commandName: distinct + - commandFailedEvent: + commandName: distinct + - commandStartedEvent: + commandName: distinct + - commandSucceededEvent: + commandName: distinct + + - + description: 'collection.find retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [find] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: find + arguments: + filter: {} + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandSucceededEvent: + commandName: find + + - + description: 'collection.findOne retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [find] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: findOne + arguments: + filter: {} + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandSucceededEvent: + commandName: find + + - + description: 'collection.listIndexes retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [listIndexes] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: listIndexes + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandSucceededEvent: + commandName: listIndexes + + - + description: 'collection.listIndexNames retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [listIndexes] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: listIndexNames + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandSucceededEvent: + commandName: listIndexes + + - + description: 'collection.createChangeStream retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [aggregate] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: createChangeStream + arguments: + pipeline: [] + saveResultAsEntity: changeStream + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandSucceededEvent: + commandName: aggregate + + - + description: 'collection.insertOne retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [insert] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: insertOne + arguments: + document: { _id: 2, x: 22 } + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandSucceededEvent: + commandName: insert + + - + description: 'collection.insertMany retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [insert] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: insertMany + arguments: + documents: + - { _id: 2, x: 22 } + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandSucceededEvent: + commandName: insert + + - + description: 'collection.deleteOne retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [delete] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: deleteOne + arguments: + filter: {} + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandSucceededEvent: + commandName: delete + + - + description: 'collection.deleteMany retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [delete] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: deleteMany + arguments: + filter: {} + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandSucceededEvent: + commandName: delete + + - + description: 'collection.replaceOne retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [update] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: replaceOne + arguments: + filter: {} + replacement: { x: 22 } + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandSucceededEvent: + commandName: update + + - + description: 'collection.updateOne retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [update] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: updateOne + arguments: + filter: {} + update: { $set: { x: 22 } } + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandSucceededEvent: + commandName: update + + - + description: 'collection.updateMany retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [update] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: updateMany + arguments: + filter: {} + update: { $set: { x: 22 } } + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandSucceededEvent: + commandName: update + + - + description: 'collection.findOneAndDelete retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [findAndModify] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: findOneAndDelete + arguments: + filter: {} + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandSucceededEvent: + commandName: findAndModify + + - + description: 'collection.findOneAndReplace retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [findAndModify] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: findOneAndReplace + arguments: + filter: {} + replacement: { x: 22 } + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandSucceededEvent: + commandName: findAndModify + + - + description: 'collection.findOneAndUpdate retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [findAndModify] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: findOneAndUpdate + arguments: + filter: {} + update: { $set: { x: 22 } } + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandSucceededEvent: + commandName: findAndModify + + - + description: 'collection.bulkWrite retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [insert] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: bulkWrite + arguments: + requests: + - insertOne: + document: { _id: 2, x: 22 } + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandSucceededEvent: + commandName: insert + + - + description: 'collection.createIndex retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [createIndexes] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: createIndex + arguments: + keys: { x: 11 } + name: "x_11" + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: createIndexes + - commandFailedEvent: + commandName: createIndexes + - commandStartedEvent: + commandName: createIndexes + - commandFailedEvent: + commandName: createIndexes + - commandStartedEvent: + commandName: createIndexes + - commandFailedEvent: + commandName: createIndexes + - commandStartedEvent: + commandName: createIndexes + - commandSucceededEvent: + commandName: createIndexes + + - + description: 'collection.dropIndex retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + - + object: *utilCollection + name: createIndex + arguments: + keys: { x: 11 } + name: "x_11" + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [dropIndexes] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: dropIndex + arguments: + name: "x_11" + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandSucceededEvent: + commandName: dropIndexes + + - + description: 'collection.dropIndexes retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [dropIndexes] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: dropIndexes + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandSucceededEvent: + commandName: dropIndexes diff --git a/source/client-backpressure/tests/backpressure-retry-loop.yml.template b/source/client-backpressure/tests/backpressure-retry-loop.yml.template new file mode 100644 index 0000000000..049cbbac3a --- /dev/null +++ b/source/client-backpressure/tests/backpressure-retry-loop.yml.template @@ -0,0 +1,128 @@ +# Tests in this file are generated from backpressure-retry-loop.yml.template. + +description: tests that operations respect overload backoff retry loop + +schemaVersion: '1.0' + +runOnRequirements: + - + minServerVersion: '4.4' # failCommand + topologies: [replicaset, sharded, load-balanced] + +createEntities: + - + client: + id: &client client + useMultipleMongoses: false + observeEvents: [ 'commandStartedEvent', 'commandSucceededEvent', 'commandFailedEvent' ] + + - + client: + id: &failPointClient failPointClient + useMultipleMongoses: false + + - + database: + id: &utilDb utilDb + client: *failPointClient + databaseName: &database_name retryable-writes-tests + + - + collection: + id: &utilCollection utilCollection + database: *utilDb + collectionName: &collection_name coll + + - + database: + id: &database database + client: *client + databaseName: &database_name retryable-writes-tests + - + collection: + id: &collection collection + database: *database + collectionName: &collection_name coll + +initialData: + - + collectionName: *collection_name + databaseName: *database_name + documents: + - { _id: 1, x: 11 } + - { _id: 2, x: 22 } + +tests: +{% for operation in operations %} + - + description: '{{operation.object}}.{{operation.operation_name}} retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + {%- if operation.operation_name == "dropIndex" %} + - + object: *utilCollection + name: createIndex + arguments: + keys: { x: 11 } + name: "x_11" + {%- endif %} + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [{{operation.command_name}}] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *{{operation.object}} + name: {{operation.operation_name}} + {%- if operation.arguments|length > 0 %} + arguments: + {%- for arg in operation.arguments %} + {{arg}} + {%- endfor -%} + {%- endif %} + {%- if operation.operation_name == "createChangeStream" %} + saveResultAsEntity: changeStream + {%- endif %} + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: {{operation.command_name}} + - commandFailedEvent: + commandName: {{operation.command_name}} + - commandStartedEvent: + commandName: {{operation.command_name}} + - commandFailedEvent: + commandName: {{operation.command_name}} + - commandStartedEvent: + commandName: {{operation.command_name}} + - commandFailedEvent: + commandName: {{operation.command_name}} + - commandStartedEvent: + commandName: {{operation.command_name}} + - commandSucceededEvent: + commandName: {{operation.command_name}} +{% endfor -%} diff --git a/source/client-backpressure/tests/backpressure-retry-max-attempts.json b/source/client-backpressure/tests/backpressure-retry-max-attempts.json new file mode 100644 index 0000000000..36afc55cc5 --- /dev/null +++ b/source/client-backpressure/tests/backpressure-retry-max-attempts.json @@ -0,0 +1,1262 @@ +{ + "description": "tests that operations retry at most maxAttempts=5 times", + "schemaVersion": "1.0", + "runOnRequirements": [ + { + "minServerVersion": "4.4", + "topologies": [ + "replicaset", + "sharded", + "load-balanced" + ] + } + ], + "createEntities": [ + { + "client": { + "id": "client", + "useMultipleMongoses": false + } + }, + { + "client": { + "id": "failPointClient", + "useMultipleMongoses": false + } + }, + { + "database": { + "id": "database", + "client": "client", + "databaseName": "retryable-writes-tests" + } + }, + { + "collection": { + "id": "collection", + "database": "database", + "collectionName": "coll" + } + } + ], + "initialData": [ + { + "collectionName": "coll", + "databaseName": "retryable-writes-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + } + ] + } + ], + "tests": [ + { + "description": "client.listDatabases retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "listDatabases" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "client", + "name": "listDatabases", + "arguments": { + "filter": {} + }, + "expectError": true + } + ] + }, + { + "description": "client.listDatabaseNames retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "listDatabases" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "client", + "name": "listDatabaseNames", + "expectError": true + } + ] + }, + { + "description": "client.createChangeStream retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "client", + "name": "createChangeStream", + "arguments": { + "pipeline": [] + }, + "saveResultAsEntity": "changeStream", + "expectError": true + } + ] + }, + { + "description": "client.clientBulkWrite retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "bulkWrite" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "client", + "name": "clientBulkWrite", + "arguments": { + "models": [ + { + "insertOne": { + "namespace": "retryable-writes-tests.coll", + "document": { + "_id": 8, + "x": 88 + } + } + } + ] + }, + "expectError": true + } + ] + }, + { + "description": "database.aggregate retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "database", + "name": "aggregate", + "arguments": { + "pipeline": [ + { + "$listLocalSessions": {} + }, + { + "$limit": 1 + } + ] + }, + "expectError": true + } + ] + }, + { + "description": "database.listCollections retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "listCollections" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "database", + "name": "listCollections", + "arguments": { + "filter": {} + }, + "expectError": true + } + ] + }, + { + "description": "database.listCollectionNames retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "listCollections" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "database", + "name": "listCollectionNames", + "arguments": { + "filter": {} + }, + "expectError": true + } + ] + }, + { + "description": "database.runCommand retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "ping" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "database", + "name": "runCommand", + "arguments": { + "command": { + "ping": 1 + }, + "commandName": "ping" + }, + "expectError": true + } + ] + }, + { + "description": "database.createChangeStream retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "database", + "name": "createChangeStream", + "arguments": { + "pipeline": [] + }, + "saveResultAsEntity": "changeStream", + "expectError": true + } + ] + }, + { + "description": "collection.aggregate retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "aggregate", + "arguments": { + "pipeline": [] + }, + "expectError": true + } + ] + }, + { + "description": "collection.countDocuments retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "countDocuments", + "arguments": { + "filter": {} + }, + "expectError": true + } + ] + }, + { + "description": "collection.estimatedDocumentCount retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "count" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "estimatedDocumentCount", + "expectError": true + } + ] + }, + { + "description": "collection.distinct retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "distinct" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "distinct", + "arguments": { + "fieldName": "x", + "filter": {} + }, + "expectError": true + } + ] + }, + { + "description": "collection.find retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "find" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "find", + "arguments": { + "filter": {} + }, + "expectError": true + } + ] + }, + { + "description": "collection.findOne retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "find" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "findOne", + "arguments": { + "filter": {} + }, + "expectError": true + } + ] + }, + { + "description": "collection.listIndexes retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "listIndexes" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "listIndexes", + "expectError": true + } + ] + }, + { + "description": "collection.listIndexNames retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "listIndexes" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "listIndexNames", + "expectError": true + } + ] + }, + { + "description": "collection.createChangeStream retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "createChangeStream", + "arguments": { + "pipeline": [] + }, + "saveResultAsEntity": "changeStream", + "expectError": true + } + ] + }, + { + "description": "collection.insertOne retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "insert" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "insertOne", + "arguments": { + "document": { + "_id": 2, + "x": 22 + } + }, + "expectError": true + } + ] + }, + { + "description": "collection.insertMany retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "insert" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "insertMany", + "arguments": { + "documents": [ + { + "_id": 2, + "x": 22 + } + ] + }, + "expectError": true + } + ] + }, + { + "description": "collection.deleteOne retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "delete" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "deleteOne", + "arguments": { + "filter": {} + }, + "expectError": true + } + ] + }, + { + "description": "collection.deleteMany retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "delete" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "deleteMany", + "arguments": { + "filter": {} + }, + "expectError": true + } + ] + }, + { + "description": "collection.replaceOne retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "update" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "replaceOne", + "arguments": { + "filter": {}, + "replacement": { + "x": 22 + } + }, + "expectError": true + } + ] + }, + { + "description": "collection.updateOne retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "update" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "updateOne", + "arguments": { + "filter": {}, + "update": { + "$set": { + "x": 22 + } + } + }, + "expectError": true + } + ] + }, + { + "description": "collection.updateMany retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "update" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "updateMany", + "arguments": { + "filter": {}, + "update": { + "$set": { + "x": 22 + } + } + }, + "expectError": true + } + ] + }, + { + "description": "collection.findOneAndDelete retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "findAndModify" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "findOneAndDelete", + "arguments": { + "filter": {} + }, + "expectError": true + } + ] + }, + { + "description": "collection.findOneAndReplace retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "findAndModify" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "findOneAndReplace", + "arguments": { + "filter": {}, + "replacement": { + "x": 22 + } + }, + "expectError": true + } + ] + }, + { + "description": "collection.findOneAndUpdate retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "findAndModify" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "findOneAndUpdate", + "arguments": { + "filter": {}, + "update": { + "$set": { + "x": 22 + } + } + }, + "expectError": true + } + ] + }, + { + "description": "collection.bulkWrite retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "insert" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "bulkWrite", + "arguments": { + "requests": [ + { + "insertOne": { + "document": { + "_id": 2, + "x": 22 + } + } + } + ] + }, + "expectError": true + } + ] + }, + { + "description": "collection.createIndex retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "createIndexes" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "createIndex", + "arguments": { + "keys": { + "x": 11 + }, + "name": "x_11" + }, + "expectError": true + } + ] + }, + { + "description": "collection.dropIndex retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "dropIndexes" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "dropIndex", + "arguments": { + "name": "x_11" + }, + "expectError": true + } + ] + }, + { + "description": "collection.dropIndexes retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "dropIndexes" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "dropIndexes", + "expectError": true + } + ] + } + ] +} diff --git a/source/client-backpressure/tests/backpressure-retry-max-attempts.yml b/source/client-backpressure/tests/backpressure-retry-max-attempts.yml new file mode 100644 index 0000000000..998b48c78e --- /dev/null +++ b/source/client-backpressure/tests/backpressure-retry-max-attempts.yml @@ -0,0 +1,753 @@ +# Tests in this file are generated from backpressure-retry-loop.yml.template. + +description: tests that operations retry at most maxAttempts=5 times + +schemaVersion: '1.0' + +runOnRequirements: + - + minServerVersion: '4.4' # failCommand + topologies: [replicaset, sharded, load-balanced] + +createEntities: + - + client: + id: &client client + useMultipleMongoses: false + + - + client: + id: &failPointClient failPointClient + useMultipleMongoses: false + + - + database: + id: &database database + client: *client + databaseName: &database_name retryable-writes-tests + - + collection: + id: &collection collection + database: *database + collectionName: &collection_name coll + +initialData: + - + collectionName: *collection_name + databaseName: *database_name + documents: + - { _id: 1, x: 11 } + - { _id: 2, x: 22 } + +tests: + + - + description: 'client.listDatabases retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [listDatabases] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *client + name: listDatabases + arguments: + filter: {} + expectError: true + + - + description: 'client.listDatabaseNames retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [listDatabases] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *client + name: listDatabaseNames + expectError: true + + - + description: 'client.createChangeStream retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [aggregate] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *client + name: createChangeStream + arguments: + pipeline: [] + saveResultAsEntity: changeStream + expectError: true + + - + description: 'client.clientBulkWrite retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [bulkWrite] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *client + name: clientBulkWrite + arguments: + models: + - insertOne: + namespace: retryable-writes-tests.coll + document: { _id: 8, x: 88 } + expectError: true + + - + description: 'database.aggregate retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [aggregate] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *database + name: aggregate + arguments: + pipeline: [ { $listLocalSessions: {} }, { $limit: 1 } ] + expectError: true + + - + description: 'database.listCollections retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [listCollections] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *database + name: listCollections + arguments: + filter: {} + expectError: true + + - + description: 'database.listCollectionNames retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [listCollections] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *database + name: listCollectionNames + arguments: + filter: {} + expectError: true + + - + description: 'database.runCommand retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [ping] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *database + name: runCommand + arguments: + command: { ping: 1 } + commandName: ping + expectError: true + + - + description: 'database.createChangeStream retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [aggregate] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *database + name: createChangeStream + arguments: + pipeline: [] + saveResultAsEntity: changeStream + expectError: true + + - + description: 'collection.aggregate retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [aggregate] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: aggregate + arguments: + pipeline: [] + expectError: true + + - + description: 'collection.countDocuments retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [aggregate] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: countDocuments + arguments: + filter: {} + expectError: true + + - + description: 'collection.estimatedDocumentCount retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [count] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: estimatedDocumentCount + expectError: true + + - + description: 'collection.distinct retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [distinct] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: distinct + arguments: + fieldName: x + filter: {} + expectError: true + + - + description: 'collection.find retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [find] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: find + arguments: + filter: {} + expectError: true + + - + description: 'collection.findOne retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [find] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: findOne + arguments: + filter: {} + expectError: true + + - + description: 'collection.listIndexes retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [listIndexes] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: listIndexes + expectError: true + + - + description: 'collection.listIndexNames retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [listIndexes] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: listIndexNames + expectError: true + + - + description: 'collection.createChangeStream retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [aggregate] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: createChangeStream + arguments: + pipeline: [] + saveResultAsEntity: changeStream + expectError: true + + - + description: 'collection.insertOne retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [insert] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: insertOne + arguments: + document: { _id: 2, x: 22 } + expectError: true + + - + description: 'collection.insertMany retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [insert] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: insertMany + arguments: + documents: + - { _id: 2, x: 22 } + expectError: true + + - + description: 'collection.deleteOne retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [delete] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: deleteOne + arguments: + filter: {} + expectError: true + + - + description: 'collection.deleteMany retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [delete] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: deleteMany + arguments: + filter: {} + expectError: true + + - + description: 'collection.replaceOne retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [update] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: replaceOne + arguments: + filter: {} + replacement: { x: 22 } + expectError: true + + - + description: 'collection.updateOne retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [update] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: updateOne + arguments: + filter: {} + update: { $set: { x: 22 } } + expectError: true + + - + description: 'collection.updateMany retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [update] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: updateMany + arguments: + filter: {} + update: { $set: { x: 22 } } + expectError: true + + - + description: 'collection.findOneAndDelete retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [findAndModify] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: findOneAndDelete + arguments: + filter: {} + expectError: true + + - + description: 'collection.findOneAndReplace retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [findAndModify] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: findOneAndReplace + arguments: + filter: {} + replacement: { x: 22 } + expectError: true + + - + description: 'collection.findOneAndUpdate retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [findAndModify] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: findOneAndUpdate + arguments: + filter: {} + update: { $set: { x: 22 } } + expectError: true + + - + description: 'collection.bulkWrite retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [insert] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: bulkWrite + arguments: + requests: + - insertOne: + document: { _id: 2, x: 22 } + expectError: true + + - + description: 'collection.createIndex retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [createIndexes] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: createIndex + arguments: + keys: { x: 11 } + name: "x_11" + expectError: true + + - + description: 'collection.dropIndex retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [dropIndexes] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: dropIndex + arguments: + name: "x_11" + expectError: true + + - + description: 'collection.dropIndexes retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [dropIndexes] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: dropIndexes + expectError: true diff --git a/source/client-backpressure/tests/backpressure-retry-max-attempts.yml.template b/source/client-backpressure/tests/backpressure-retry-max-attempts.yml.template new file mode 100644 index 0000000000..bf089211fd --- /dev/null +++ b/source/client-backpressure/tests/backpressure-retry-max-attempts.yml.template @@ -0,0 +1,72 @@ +# Tests in this file are generated from backpressure-retry-max-attempts.yml.template. + +description: tests that operations retry at most maxAttempts=5 times + +schemaVersion: '1.0' + +runOnRequirements: + - + minServerVersion: '4.4' # failCommand + topologies: [replicaset, sharded, load-balanced] + +createEntities: + - + client: + id: &client client + useMultipleMongoses: false + + - + client: + id: &failPointClient failPointClient + useMultipleMongoses: false + + - + database: + id: &database database + client: *client + databaseName: &database_name retryable-writes-tests + - + collection: + id: &collection collection + database: *database + collectionName: &collection_name coll + +initialData: + - + collectionName: *collection_name + databaseName: *database_name + documents: + - { _id: 1, x: 11 } + - { _id: 2, x: 22 } + +tests: +{% for operation in operations %} + - + description: '{{operation.object}}.{{operation.operation_name}} retries at most maxAttempts=5 times' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [{{operation.command_name}}] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *{{operation.object}} + name: {{operation.operation_name}} + {%- if operation.arguments|length > 0 %} + arguments: + {%- for arg in operation.arguments %} + {{arg}} + {%- endfor -%} + {%- endif %} + {%- if operation.operation_name == "createChangeStream" %} + saveResultAsEntity: changeStream + {%- endif %} + expectError: true +{% endfor -%} diff --git a/source/etc/generate-backpressure-retryability-tests.py b/source/etc/generate-backpressure-retryability-tests.py new file mode 100644 index 0000000000..305cfa585d --- /dev/null +++ b/source/etc/generate-backpressure-retryability-tests.py @@ -0,0 +1,125 @@ +from collections import namedtuple +from jinja2 import Template +import os +import sys + +Operation = namedtuple( + 'Operation', ['operation_name', 'command_name', 'object', 'arguments']) + +CLIENT_BULK_WRITE_ARGUMENTS = '''models: + - insertOne: + namespace: retryable-writes-tests.coll + document: { _id: 8, x: 88 }''' + +CLIENT_OPERATIONS = [ + Operation('listDatabases', 'listDatabases', 'client', ['filter: {}']), + Operation('listDatabaseNames', 'listDatabases', 'client', []), + Operation('createChangeStream', 'aggregate', 'client', ['pipeline: []']), + Operation('clientBulkWrite', 'bulkWrite', 'client', [CLIENT_BULK_WRITE_ARGUMENTS]) +] + +RUN_COMMAND_ARGUMENTS = '''command: { ping: 1 } + commandName: ping''' + +DB_OPERATIONS = [ + Operation('aggregate', 'aggregate', 'database', [ + 'pipeline: [ { $listLocalSessions: {} }, { $limit: 1 } ]']), + Operation('listCollections', 'listCollections', + 'database', ['filter: {}']), + Operation('listCollectionNames', 'listCollections', + 'database', ['filter: {}']), # Optional. + Operation('runCommand', 'ping', 'database', [RUN_COMMAND_ARGUMENTS]), + Operation('createChangeStream', 'aggregate', 'database', ['pipeline: []']) +] + +INSERT_MANY_ARGUMENTS = '''documents: + - { _id: 2, x: 22 }''' + +BULK_WRITE_ARGUMENTS = '''requests: + - insertOne: + document: { _id: 2, x: 22 }''' + +COLLECTION_READ_OPERATIONS = [ + Operation('aggregate', 'aggregate', 'collection', ['pipeline: []']), + # Operation('count', 'count', 'collection', ['filter: {}']), # Deprecated. + Operation('countDocuments', 'aggregate', 'collection', ['filter: {}']), + Operation('estimatedDocumentCount', 'count', 'collection', []), + Operation('distinct', 'distinct', 'collection', + ['fieldName: x', 'filter: {}']), + Operation('find', 'find', 'collection', ['filter: {}']), + Operation('findOne', 'find', 'collection', ['filter: {}']), # Optional. + Operation('listIndexes', 'listIndexes', 'collection', []), + Operation('listIndexNames', 'listIndexes', 'collection', []), # Optional. + Operation('createChangeStream', 'aggregate', + 'collection', ['pipeline: []']), +] + +COLLECTION_WRITE_OPERATIONS = [ + Operation('insertOne', 'insert', 'collection', + ['document: { _id: 2, x: 22 }']), + Operation('insertMany', 'insert', 'collection', [INSERT_MANY_ARGUMENTS]), + Operation('deleteOne', 'delete', 'collection', ['filter: {}']), + Operation('deleteMany', 'delete', 'collection', ['filter: {}']), + Operation('replaceOne', 'update', 'collection', [ + 'filter: {}', 'replacement: { x: 22 }']), + Operation('updateOne', 'update', 'collection', [ + 'filter: {}', 'update: { $set: { x: 22 } }']), + Operation('updateMany', 'update', 'collection', [ + 'filter: {}', 'update: { $set: { x: 22 } }']), + Operation('findOneAndDelete', 'findAndModify', + 'collection', ['filter: {}']), + Operation('findOneAndReplace', 'findAndModify', 'collection', + ['filter: {}', 'replacement: { x: 22 }']), + Operation('findOneAndUpdate', 'findAndModify', 'collection', + ['filter: {}', 'update: { $set: { x: 22 } }']), + Operation('bulkWrite', 'insert', 'collection', [BULK_WRITE_ARGUMENTS]), + Operation('createIndex', 'createIndexes', 'collection', + ['keys: { x: 11 }', 'name: "x_11"']), + Operation('dropIndex', 'dropIndexes', 'collection', ['name: "x_11"']), + Operation('dropIndexes', 'dropIndexes', 'collection', []), +] + +COLLECTION_OPERATIONS = COLLECTION_READ_OPERATIONS + COLLECTION_WRITE_OPERATIONS + +# Session and GridFS operations are generally tested in other files, so they're not included in the list of all +# operations. Individual generation functions can choose to include them if needed. +OPERATIONS = CLIENT_OPERATIONS + DB_OPERATIONS + COLLECTION_OPERATIONS + +# ./source/etc +DIR = os.path.dirname(os.path.realpath(__file__)) + + +def get_template(file, templates_dir): + path = f'{templates_dir}/{file}.yml.template' + return Template(open(path, 'r').read()) + + +def write_yaml(file, template, tests_dir, injections): + rendered = template.render(**injections) + path = f'{tests_dir}/{file}.yml' + open(path, 'w').write(rendered) + + +def generate(name, templates_dir, tests_dir, operations): + template = get_template(name, templates_dir) + injections = { + 'operations': operations, + } + write_yaml(name, template, tests_dir, injections) + + +def generate_retry_loop_tests(): + templates_dir = f'{os.path.dirname(DIR)}/client-backpressure/tests' + tests_dir = f'{os.path.dirname(DIR)}/client-backpressure/tests' + generate('backpressure-retry-loop', templates_dir, + tests_dir, OPERATIONS) + + +def generate_max_attempts_tests(): + templates_dir = f'{os.path.dirname(DIR)}/client-backpressure/tests' + tests_dir = f'{os.path.dirname(DIR)}/client-backpressure/tests' + generate('backpressure-retry-max-attempts', templates_dir, + tests_dir, OPERATIONS) + +generate_retry_loop_tests() +generate_max_attempts_tests() \ No newline at end of file From 072b45371972be8d136753ae311a039e3c42d9f0 Mon Sep 17 00:00:00 2001 From: bailey Date: Mon, 1 Dec 2025 16:39:22 -0700 Subject: [PATCH 05/12] test fixes and add prose test - add prose test - add assertions on the number of retries for maxAttempts tests - don't run clientBulkWrite tests on <8.0 servers --- source/client-backpressure/tests/README.md | 46 + .../tests/backpressure-retry-loop.json | 3 + .../tests/backpressure-retry-loop.yml | 2 + .../backpressure-retry-loop.yml.template | 4 + .../backpressure-retry-max-attempts.json | 2408 +++++++++++++++-- .../tests/backpressure-retry-max-attempts.yml | 1125 +++++++- ...ckpressure-retry-max-attempts.yml.template | 38 +- 7 files changed, 3400 insertions(+), 226 deletions(-) diff --git a/source/client-backpressure/tests/README.md b/source/client-backpressure/tests/README.md index 7a70e4ad76..a4e62b9ec1 100644 --- a/source/client-backpressure/tests/README.md +++ b/source/client-backpressure/tests/README.md @@ -10,6 +10,52 @@ retryable reads. These tests utilize the [Unified Test Format](../../unified-tes Several prose tests, which are not easily expressed in YAML, are also presented in this file. Those tests will need to be manually implemented by each driver. +### Prose Tests + +#### Test 1: Operation Retry Uses Exponential Backoff + +Drivers should test that retries do not occur immediately when a SystemOverloadedError is encountered. + +1. let `client` be a `MongoClient` +2. let `collection` be a collection +3. Now, run transactions without backoff: + 1. Configure the random number generator used for jitter to always return `0` -- this effectively disables backoff. + + 2. Configure the following failPoint: + + ```javascript + { + configureFailPoint: 'failCommand', + mode: 'alwaysOn', + data: { + failCommands: ['insert'], + errorCode: 2, + errorLabels: ['SystemOverloadedError', 'RetryableError'] + } + } + ``` + + 3. Execute the following command. Expect that the command errors. Measure the duration of the command execution. + + ```javascript + const start = performance.now(); + expect( + await coll.insertOne({ a: 1 }).catch(e => e) + ).to.be.an.instanceof(MongoServerError); + const end = performance.now(); + ``` + + 4. Configure the random number generator used for jitter to always return `1`. + + 5. Execute step 3 again. + + 6. Compare the two time between the two runs. + ```python + assertTrue(absolute_value(with_backoff_time - (no_backoff_time + 3.1 seconds)) < 1) + ``` + The sum of 5 backoffs is 3.1 seconds. There is a 1-second window to account for potential variance between the two + runs. + ## Changelog - 2025-XX-XX: Initial version. diff --git a/source/client-backpressure/tests/backpressure-retry-loop.json b/source/client-backpressure/tests/backpressure-retry-loop.json index 21fc802344..ae944f73ad 100644 --- a/source/client-backpressure/tests/backpressure-retry-loop.json +++ b/source/client-backpressure/tests/backpressure-retry-loop.json @@ -381,6 +381,9 @@ }, { "description": "client.clientBulkWrite retries using operation loop", + "runOnRequirements": { + "minServerVersion": "8.0" + }, "operations": [ { "object": "utilCollection", diff --git a/source/client-backpressure/tests/backpressure-retry-loop.yml b/source/client-backpressure/tests/backpressure-retry-loop.yml index ec2fa25b8f..c612986233 100644 --- a/source/client-backpressure/tests/backpressure-retry-loop.yml +++ b/source/client-backpressure/tests/backpressure-retry-loop.yml @@ -223,6 +223,8 @@ tests: - description: 'client.clientBulkWrite retries using operation loop' + runOnRequirements: + minServerVersion: '8.0' operations: - object: *utilCollection diff --git a/source/client-backpressure/tests/backpressure-retry-loop.yml.template b/source/client-backpressure/tests/backpressure-retry-loop.yml.template index 049cbbac3a..cad83625d3 100644 --- a/source/client-backpressure/tests/backpressure-retry-loop.yml.template +++ b/source/client-backpressure/tests/backpressure-retry-loop.yml.template @@ -56,6 +56,10 @@ tests: {% for operation in operations %} - description: '{{operation.object}}.{{operation.operation_name}} retries using operation loop' + {%- if ((operation.operation_name == 'clientBulkWrite')) %} + runOnRequirements: + minServerVersion: '8.0' + {%- endif %} operations: - object: *utilCollection diff --git a/source/client-backpressure/tests/backpressure-retry-max-attempts.json b/source/client-backpressure/tests/backpressure-retry-max-attempts.json index 36afc55cc5..7e9cc67d7a 100644 --- a/source/client-backpressure/tests/backpressure-retry-max-attempts.json +++ b/source/client-backpressure/tests/backpressure-retry-max-attempts.json @@ -15,7 +15,12 @@ { "client": { "id": "client", - "useMultipleMongoses": false + "useMultipleMongoses": false, + "observeEvents": [ + "commandStartedEvent", + "commandSucceededEvent", + "commandFailedEvent" + ] } }, { @@ -57,7 +62,7 @@ ], "tests": [ { - "description": "client.listDatabases retries at most maxAttempts times (maxAttempts=5)", + "description": "client.listDatabases retries at most maxAttempts=5 times", "operations": [ { "name": "failPoint", @@ -66,9 +71,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "listDatabases" @@ -90,10 +93,77 @@ }, "expectError": true } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + } + ] + } ] }, { - "description": "client.listDatabaseNames retries at most maxAttempts times (maxAttempts=5)", + "description": "client.listDatabaseNames retries at most maxAttempts=5 times", "operations": [ { "name": "failPoint", @@ -102,9 +172,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "listDatabases" @@ -123,10 +191,77 @@ "name": "listDatabaseNames", "expectError": true } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + } + ] + } ] }, { - "description": "client.createChangeStream retries at most maxAttempts times (maxAttempts=5)", + "description": "client.createChangeStream retries at most maxAttempts=5 times", "operations": [ { "name": "failPoint", @@ -135,9 +270,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "aggregate" @@ -160,10 +293,80 @@ "saveResultAsEntity": "changeStream", "expectError": true } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + } + ] + } ] }, { - "description": "client.clientBulkWrite retries at most maxAttempts times (maxAttempts=5)", + "description": "client.clientBulkWrite retries at most maxAttempts=5 times", + "runOnRequirements": { + "minServerVersion": "8.0" + }, "operations": [ { "name": "failPoint", @@ -172,9 +375,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "bulkWrite" @@ -206,10 +407,77 @@ }, "expectError": true } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandFailedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandStartedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandFailedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandStartedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandFailedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandStartedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandFailedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandStartedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandFailedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandStartedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandFailedEvent": { + "commandName": "bulkWrite" + } + } + ] + } ] }, { - "description": "database.aggregate retries at most maxAttempts times (maxAttempts=5)", + "description": "database.aggregate retries at most maxAttempts=5 times", "operations": [ { "name": "failPoint", @@ -218,9 +486,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "aggregate" @@ -249,10 +515,77 @@ }, "expectError": true } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + } + ] + } ] }, { - "description": "database.listCollections retries at most maxAttempts times (maxAttempts=5)", + "description": "database.listCollections retries at most maxAttempts=5 times", "operations": [ { "name": "failPoint", @@ -261,9 +594,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "listCollections" @@ -285,10 +616,77 @@ }, "expectError": true } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + } + ] + } ] }, { - "description": "database.listCollectionNames retries at most maxAttempts times (maxAttempts=5)", + "description": "database.listCollectionNames retries at most maxAttempts=5 times", "operations": [ { "name": "failPoint", @@ -297,9 +695,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "listCollections" @@ -321,10 +717,77 @@ }, "expectError": true } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + } + ] + } ] }, { - "description": "database.runCommand retries at most maxAttempts times (maxAttempts=5)", + "description": "database.runCommand retries at most maxAttempts=5 times", "operations": [ { "name": "failPoint", @@ -333,9 +796,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "ping" @@ -360,10 +821,77 @@ }, "expectError": true } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "ping" + } + }, + { + "commandFailedEvent": { + "commandName": "ping" + } + }, + { + "commandStartedEvent": { + "commandName": "ping" + } + }, + { + "commandFailedEvent": { + "commandName": "ping" + } + }, + { + "commandStartedEvent": { + "commandName": "ping" + } + }, + { + "commandFailedEvent": { + "commandName": "ping" + } + }, + { + "commandStartedEvent": { + "commandName": "ping" + } + }, + { + "commandFailedEvent": { + "commandName": "ping" + } + }, + { + "commandStartedEvent": { + "commandName": "ping" + } + }, + { + "commandFailedEvent": { + "commandName": "ping" + } + }, + { + "commandStartedEvent": { + "commandName": "ping" + } + }, + { + "commandFailedEvent": { + "commandName": "ping" + } + } + ] + } ] }, { - "description": "database.createChangeStream retries at most maxAttempts times (maxAttempts=5)", + "description": "database.createChangeStream retries at most maxAttempts=5 times", "operations": [ { "name": "failPoint", @@ -372,9 +900,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "aggregate" @@ -397,10 +923,77 @@ "saveResultAsEntity": "changeStream", "expectError": true } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + } + ] + } ] }, { - "description": "collection.aggregate retries at most maxAttempts times (maxAttempts=5)", + "description": "collection.aggregate retries at most maxAttempts=5 times", "operations": [ { "name": "failPoint", @@ -409,9 +1002,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "aggregate" @@ -433,29 +1024,94 @@ }, "expectError": true } - ] - }, - { - "description": "collection.countDocuments retries at most maxAttempts times (maxAttempts=5)", - "operations": [ - { - "name": "failPoint", - "object": "testRunner", - "arguments": { - "client": "failPointClient", - "failPoint": { - "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, - "data": { - "failCommands": [ - "aggregate" - ], - "errorLabels": [ - "RetryableError", - "SystemOverloadedError" - ], + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + } + ] + } + ] + }, + { + "description": "collection.countDocuments retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], "errorCode": 2 } } @@ -469,10 +1125,77 @@ }, "expectError": true } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + } + ] + } ] }, { - "description": "collection.estimatedDocumentCount retries at most maxAttempts times (maxAttempts=5)", + "description": "collection.estimatedDocumentCount retries at most maxAttempts=5 times", "operations": [ { "name": "failPoint", @@ -481,9 +1204,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "count" @@ -502,10 +1223,77 @@ "name": "estimatedDocumentCount", "expectError": true } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "count" + } + }, + { + "commandFailedEvent": { + "commandName": "count" + } + }, + { + "commandStartedEvent": { + "commandName": "count" + } + }, + { + "commandFailedEvent": { + "commandName": "count" + } + }, + { + "commandStartedEvent": { + "commandName": "count" + } + }, + { + "commandFailedEvent": { + "commandName": "count" + } + }, + { + "commandStartedEvent": { + "commandName": "count" + } + }, + { + "commandFailedEvent": { + "commandName": "count" + } + }, + { + "commandStartedEvent": { + "commandName": "count" + } + }, + { + "commandFailedEvent": { + "commandName": "count" + } + }, + { + "commandStartedEvent": { + "commandName": "count" + } + }, + { + "commandFailedEvent": { + "commandName": "count" + } + } + ] + } ] }, { - "description": "collection.distinct retries at most maxAttempts times (maxAttempts=5)", + "description": "collection.distinct retries at most maxAttempts=5 times", "operations": [ { "name": "failPoint", @@ -514,9 +1302,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "distinct" @@ -539,10 +1325,77 @@ }, "expectError": true } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "distinct" + } + }, + { + "commandFailedEvent": { + "commandName": "distinct" + } + }, + { + "commandStartedEvent": { + "commandName": "distinct" + } + }, + { + "commandFailedEvent": { + "commandName": "distinct" + } + }, + { + "commandStartedEvent": { + "commandName": "distinct" + } + }, + { + "commandFailedEvent": { + "commandName": "distinct" + } + }, + { + "commandStartedEvent": { + "commandName": "distinct" + } + }, + { + "commandFailedEvent": { + "commandName": "distinct" + } + }, + { + "commandStartedEvent": { + "commandName": "distinct" + } + }, + { + "commandFailedEvent": { + "commandName": "distinct" + } + }, + { + "commandStartedEvent": { + "commandName": "distinct" + } + }, + { + "commandFailedEvent": { + "commandName": "distinct" + } + } + ] + } ] }, { - "description": "collection.find retries at most maxAttempts times (maxAttempts=5)", + "description": "collection.find retries at most maxAttempts=5 times", "operations": [ { "name": "failPoint", @@ -551,9 +1404,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "find" @@ -575,10 +1426,77 @@ }, "expectError": true } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + } + ] + } ] }, { - "description": "collection.findOne retries at most maxAttempts times (maxAttempts=5)", + "description": "collection.findOne retries at most maxAttempts=5 times", "operations": [ { "name": "failPoint", @@ -587,9 +1505,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "find" @@ -611,10 +1527,77 @@ }, "expectError": true } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + } + ] + } ] }, { - "description": "collection.listIndexes retries at most maxAttempts times (maxAttempts=5)", + "description": "collection.listIndexes retries at most maxAttempts=5 times", "operations": [ { "name": "failPoint", @@ -623,9 +1606,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "listIndexes" @@ -644,10 +1625,77 @@ "name": "listIndexes", "expectError": true } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + } + ] + } ] }, { - "description": "collection.listIndexNames retries at most maxAttempts times (maxAttempts=5)", + "description": "collection.listIndexNames retries at most maxAttempts=5 times", "operations": [ { "name": "failPoint", @@ -656,9 +1704,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "listIndexes" @@ -677,10 +1723,77 @@ "name": "listIndexNames", "expectError": true } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + } + ] + } ] }, { - "description": "collection.createChangeStream retries at most maxAttempts times (maxAttempts=5)", + "description": "collection.createChangeStream retries at most maxAttempts=5 times", "operations": [ { "name": "failPoint", @@ -689,9 +1802,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "aggregate" @@ -714,10 +1825,77 @@ "saveResultAsEntity": "changeStream", "expectError": true } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + } + ] + } ] }, { - "description": "collection.insertOne retries at most maxAttempts times (maxAttempts=5)", + "description": "collection.insertOne retries at most maxAttempts=5 times", "operations": [ { "name": "failPoint", @@ -726,9 +1904,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "insert" @@ -753,10 +1929,77 @@ }, "expectError": true } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + } + ] + } ] }, { - "description": "collection.insertMany retries at most maxAttempts times (maxAttempts=5)", + "description": "collection.insertMany retries at most maxAttempts=5 times", "operations": [ { "name": "failPoint", @@ -765,9 +2008,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "insert" @@ -794,10 +2035,77 @@ }, "expectError": true } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + } + ] + } ] }, { - "description": "collection.deleteOne retries at most maxAttempts times (maxAttempts=5)", + "description": "collection.deleteOne retries at most maxAttempts=5 times", "operations": [ { "name": "failPoint", @@ -806,9 +2114,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "delete" @@ -830,10 +2136,77 @@ }, "expectError": true } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + } + ] + } ] }, { - "description": "collection.deleteMany retries at most maxAttempts times (maxAttempts=5)", + "description": "collection.deleteMany retries at most maxAttempts=5 times", "operations": [ { "name": "failPoint", @@ -842,9 +2215,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "delete" @@ -866,10 +2237,77 @@ }, "expectError": true } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + } + ] + } ] }, { - "description": "collection.replaceOne retries at most maxAttempts times (maxAttempts=5)", + "description": "collection.replaceOne retries at most maxAttempts=5 times", "operations": [ { "name": "failPoint", @@ -878,9 +2316,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "update" @@ -892,23 +2328,90 @@ "errorCode": 2 } } - } - }, - { - "object": "collection", - "name": "replaceOne", - "arguments": { - "filter": {}, - "replacement": { - "x": 22 - } - }, - "expectError": true + } + }, + { + "object": "collection", + "name": "replaceOne", + "arguments": { + "filter": {}, + "replacement": { + "x": 22 + } + }, + "expectError": true + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + } + ] } ] }, { - "description": "collection.updateOne retries at most maxAttempts times (maxAttempts=5)", + "description": "collection.updateOne retries at most maxAttempts=5 times", "operations": [ { "name": "failPoint", @@ -917,9 +2420,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "update" @@ -946,10 +2447,77 @@ }, "expectError": true } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + } + ] + } ] }, { - "description": "collection.updateMany retries at most maxAttempts times (maxAttempts=5)", + "description": "collection.updateMany retries at most maxAttempts=5 times", "operations": [ { "name": "failPoint", @@ -958,9 +2526,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "update" @@ -987,10 +2553,77 @@ }, "expectError": true } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + } + ] + } ] }, { - "description": "collection.findOneAndDelete retries at most maxAttempts times (maxAttempts=5)", + "description": "collection.findOneAndDelete retries at most maxAttempts=5 times", "operations": [ { "name": "failPoint", @@ -999,9 +2632,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "findAndModify" @@ -1023,10 +2654,77 @@ }, "expectError": true } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + } + ] + } ] }, { - "description": "collection.findOneAndReplace retries at most maxAttempts times (maxAttempts=5)", + "description": "collection.findOneAndReplace retries at most maxAttempts=5 times", "operations": [ { "name": "failPoint", @@ -1035,9 +2733,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "findAndModify" @@ -1062,10 +2758,77 @@ }, "expectError": true } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + } + ] + } ] }, { - "description": "collection.findOneAndUpdate retries at most maxAttempts times (maxAttempts=5)", + "description": "collection.findOneAndUpdate retries at most maxAttempts=5 times", "operations": [ { "name": "failPoint", @@ -1074,9 +2837,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "findAndModify" @@ -1103,10 +2864,77 @@ }, "expectError": true } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + } + ] + } ] }, { - "description": "collection.bulkWrite retries at most maxAttempts times (maxAttempts=5)", + "description": "collection.bulkWrite retries at most maxAttempts=5 times", "operations": [ { "name": "failPoint", @@ -1115,9 +2943,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "insert" @@ -1148,10 +2974,77 @@ }, "expectError": true } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + } + ] + } ] }, { - "description": "collection.createIndex retries at most maxAttempts times (maxAttempts=5)", + "description": "collection.createIndex retries at most maxAttempts=5 times", "operations": [ { "name": "failPoint", @@ -1160,9 +3053,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "createIndexes" @@ -1187,10 +3078,77 @@ }, "expectError": true } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "createIndexes" + } + } + ] + } ] }, { - "description": "collection.dropIndex retries at most maxAttempts times (maxAttempts=5)", + "description": "collection.dropIndex retries at most maxAttempts=5 times", "operations": [ { "name": "failPoint", @@ -1199,9 +3157,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "dropIndexes" @@ -1223,10 +3179,77 @@ }, "expectError": true } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + } + ] + } ] }, { - "description": "collection.dropIndexes retries at most maxAttempts times (maxAttempts=5)", + "description": "collection.dropIndexes retries at most maxAttempts=5 times", "operations": [ { "name": "failPoint", @@ -1235,9 +3258,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "dropIndexes" @@ -1256,6 +3277,73 @@ "name": "dropIndexes", "expectError": true } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + } + ] + } ] } ] diff --git a/source/client-backpressure/tests/backpressure-retry-max-attempts.yml b/source/client-backpressure/tests/backpressure-retry-max-attempts.yml index 998b48c78e..9161828e99 100644 --- a/source/client-backpressure/tests/backpressure-retry-max-attempts.yml +++ b/source/client-backpressure/tests/backpressure-retry-max-attempts.yml @@ -1,4 +1,4 @@ -# Tests in this file are generated from backpressure-retry-loop.yml.template. +# Tests in this file are generated from backpressure-retry-max-attempts.yml.template. description: tests that operations retry at most maxAttempts=5 times @@ -14,6 +14,7 @@ createEntities: client: id: &client client useMultipleMongoses: false + observeEvents: [ 'commandStartedEvent', 'commandSucceededEvent', 'commandFailedEvent' ] - client: @@ -42,7 +43,7 @@ initialData: tests: - - description: 'client.listDatabases retries at most maxAttempts times (maxAttempts=5)' + description: 'client.listDatabases retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -50,7 +51,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [listDatabases] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -63,8 +64,39 @@ tests: filter: {} expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + + - - description: 'client.listDatabaseNames retries at most maxAttempts times (maxAttempts=5)' + description: 'client.listDatabaseNames retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -72,7 +104,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [listDatabases] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -83,8 +115,39 @@ tests: name: listDatabaseNames expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + + - - description: 'client.createChangeStream retries at most maxAttempts times (maxAttempts=5)' + description: 'client.createChangeStream retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -92,7 +155,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [aggregate] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -106,8 +169,41 @@ tests: saveResultAsEntity: changeStream expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + + - - description: 'client.clientBulkWrite retries at most maxAttempts times (maxAttempts=5)' + description: 'client.clientBulkWrite retries at most maxAttempts=5 times' + runOnRequirements: + minServerVersion: '8.0' operations: - name: failPoint object: testRunner @@ -115,7 +211,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [bulkWrite] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -131,8 +227,39 @@ tests: document: { _id: 8, x: 88 } expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: bulkWrite + - commandFailedEvent: + commandName: bulkWrite + - commandStartedEvent: + commandName: bulkWrite + - commandFailedEvent: + commandName: bulkWrite + - commandStartedEvent: + commandName: bulkWrite + - commandFailedEvent: + commandName: bulkWrite + - commandStartedEvent: + commandName: bulkWrite + - commandFailedEvent: + commandName: bulkWrite + - commandStartedEvent: + commandName: bulkWrite + - commandFailedEvent: + commandName: bulkWrite + - commandStartedEvent: + commandName: bulkWrite + - commandFailedEvent: + commandName: bulkWrite + + - - description: 'database.aggregate retries at most maxAttempts times (maxAttempts=5)' + description: 'database.aggregate retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -140,7 +267,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [aggregate] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -153,8 +280,39 @@ tests: pipeline: [ { $listLocalSessions: {} }, { $limit: 1 } ] expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + + - - description: 'database.listCollections retries at most maxAttempts times (maxAttempts=5)' + description: 'database.listCollections retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -162,7 +320,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [listCollections] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -175,8 +333,39 @@ tests: filter: {} expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + + - - description: 'database.listCollectionNames retries at most maxAttempts times (maxAttempts=5)' + description: 'database.listCollectionNames retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -184,7 +373,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [listCollections] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -197,8 +386,39 @@ tests: filter: {} expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + + - - description: 'database.runCommand retries at most maxAttempts times (maxAttempts=5)' + description: 'database.runCommand retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -206,7 +426,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [ping] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -220,8 +440,39 @@ tests: commandName: ping expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: ping + - commandFailedEvent: + commandName: ping + - commandStartedEvent: + commandName: ping + - commandFailedEvent: + commandName: ping + - commandStartedEvent: + commandName: ping + - commandFailedEvent: + commandName: ping + - commandStartedEvent: + commandName: ping + - commandFailedEvent: + commandName: ping + - commandStartedEvent: + commandName: ping + - commandFailedEvent: + commandName: ping + - commandStartedEvent: + commandName: ping + - commandFailedEvent: + commandName: ping + + - - description: 'database.createChangeStream retries at most maxAttempts times (maxAttempts=5)' + description: 'database.createChangeStream retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -229,7 +480,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [aggregate] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -243,8 +494,39 @@ tests: saveResultAsEntity: changeStream expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + + - - description: 'collection.aggregate retries at most maxAttempts times (maxAttempts=5)' + description: 'collection.aggregate retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -252,7 +534,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [aggregate] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -265,8 +547,39 @@ tests: pipeline: [] expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + + - - description: 'collection.countDocuments retries at most maxAttempts times (maxAttempts=5)' + description: 'collection.countDocuments retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -274,7 +587,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [aggregate] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -287,8 +600,39 @@ tests: filter: {} expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + + - - description: 'collection.estimatedDocumentCount retries at most maxAttempts times (maxAttempts=5)' + description: 'collection.estimatedDocumentCount retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -296,7 +640,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [count] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -307,8 +651,39 @@ tests: name: estimatedDocumentCount expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: count + - commandFailedEvent: + commandName: count + - commandStartedEvent: + commandName: count + - commandFailedEvent: + commandName: count + - commandStartedEvent: + commandName: count + - commandFailedEvent: + commandName: count + - commandStartedEvent: + commandName: count + - commandFailedEvent: + commandName: count + - commandStartedEvent: + commandName: count + - commandFailedEvent: + commandName: count + - commandStartedEvent: + commandName: count + - commandFailedEvent: + commandName: count + + - - description: 'collection.distinct retries at most maxAttempts times (maxAttempts=5)' + description: 'collection.distinct retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -316,7 +691,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [distinct] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -330,8 +705,39 @@ tests: filter: {} expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: distinct + - commandFailedEvent: + commandName: distinct + - commandStartedEvent: + commandName: distinct + - commandFailedEvent: + commandName: distinct + - commandStartedEvent: + commandName: distinct + - commandFailedEvent: + commandName: distinct + - commandStartedEvent: + commandName: distinct + - commandFailedEvent: + commandName: distinct + - commandStartedEvent: + commandName: distinct + - commandFailedEvent: + commandName: distinct + - commandStartedEvent: + commandName: distinct + - commandFailedEvent: + commandName: distinct + + - - description: 'collection.find retries at most maxAttempts times (maxAttempts=5)' + description: 'collection.find retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -339,7 +745,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [find] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -352,8 +758,39 @@ tests: filter: {} expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + + - - description: 'collection.findOne retries at most maxAttempts times (maxAttempts=5)' + description: 'collection.findOne retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -361,7 +798,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [find] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -374,8 +811,39 @@ tests: filter: {} expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + + - - description: 'collection.listIndexes retries at most maxAttempts times (maxAttempts=5)' + description: 'collection.listIndexes retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -383,7 +851,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [listIndexes] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -394,8 +862,39 @@ tests: name: listIndexes expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + + - - description: 'collection.listIndexNames retries at most maxAttempts times (maxAttempts=5)' + description: 'collection.listIndexNames retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -403,7 +902,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [listIndexes] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -414,8 +913,39 @@ tests: name: listIndexNames expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + + - - description: 'collection.createChangeStream retries at most maxAttempts times (maxAttempts=5)' + description: 'collection.createChangeStream retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -423,7 +953,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [aggregate] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -437,8 +967,39 @@ tests: saveResultAsEntity: changeStream expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + + - - description: 'collection.insertOne retries at most maxAttempts times (maxAttempts=5)' + description: 'collection.insertOne retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -446,7 +1007,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [insert] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -459,8 +1020,39 @@ tests: document: { _id: 2, x: 22 } expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + + - - description: 'collection.insertMany retries at most maxAttempts times (maxAttempts=5)' + description: 'collection.insertMany retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -468,7 +1060,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [insert] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -482,8 +1074,39 @@ tests: - { _id: 2, x: 22 } expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + + - - description: 'collection.deleteOne retries at most maxAttempts times (maxAttempts=5)' + description: 'collection.deleteOne retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -491,7 +1114,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [delete] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -504,8 +1127,39 @@ tests: filter: {} expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + + - - description: 'collection.deleteMany retries at most maxAttempts times (maxAttempts=5)' + description: 'collection.deleteMany retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -513,7 +1167,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [delete] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -526,8 +1180,39 @@ tests: filter: {} expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + + - - description: 'collection.replaceOne retries at most maxAttempts times (maxAttempts=5)' + description: 'collection.replaceOne retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -535,7 +1220,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [update] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -549,8 +1234,39 @@ tests: replacement: { x: 22 } expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + + - - description: 'collection.updateOne retries at most maxAttempts times (maxAttempts=5)' + description: 'collection.updateOne retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -558,7 +1274,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [update] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -572,8 +1288,39 @@ tests: update: { $set: { x: 22 } } expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + + - - description: 'collection.updateMany retries at most maxAttempts times (maxAttempts=5)' + description: 'collection.updateMany retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -581,7 +1328,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [update] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -595,8 +1342,39 @@ tests: update: { $set: { x: 22 } } expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + + - - description: 'collection.findOneAndDelete retries at most maxAttempts times (maxAttempts=5)' + description: 'collection.findOneAndDelete retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -604,7 +1382,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [findAndModify] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -617,8 +1395,39 @@ tests: filter: {} expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + + - - description: 'collection.findOneAndReplace retries at most maxAttempts times (maxAttempts=5)' + description: 'collection.findOneAndReplace retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -626,7 +1435,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [findAndModify] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -640,8 +1449,39 @@ tests: replacement: { x: 22 } expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + + - - description: 'collection.findOneAndUpdate retries at most maxAttempts times (maxAttempts=5)' + description: 'collection.findOneAndUpdate retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -649,7 +1489,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [findAndModify] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -663,8 +1503,39 @@ tests: update: { $set: { x: 22 } } expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + + - - description: 'collection.bulkWrite retries at most maxAttempts times (maxAttempts=5)' + description: 'collection.bulkWrite retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -672,7 +1543,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [insert] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -687,8 +1558,39 @@ tests: document: { _id: 2, x: 22 } expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + + - - description: 'collection.createIndex retries at most maxAttempts times (maxAttempts=5)' + description: 'collection.createIndex retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -696,7 +1598,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [createIndexes] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -710,8 +1612,39 @@ tests: name: "x_11" expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: createIndexes + - commandFailedEvent: + commandName: createIndexes + - commandStartedEvent: + commandName: createIndexes + - commandFailedEvent: + commandName: createIndexes + - commandStartedEvent: + commandName: createIndexes + - commandFailedEvent: + commandName: createIndexes + - commandStartedEvent: + commandName: createIndexes + - commandFailedEvent: + commandName: createIndexes + - commandStartedEvent: + commandName: createIndexes + - commandFailedEvent: + commandName: createIndexes + - commandStartedEvent: + commandName: createIndexes + - commandFailedEvent: + commandName: createIndexes + + - - description: 'collection.dropIndex retries at most maxAttempts times (maxAttempts=5)' + description: 'collection.dropIndex retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -719,7 +1652,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [dropIndexes] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -732,8 +1665,39 @@ tests: name: "x_11" expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + + - - description: 'collection.dropIndexes retries at most maxAttempts times (maxAttempts=5)' + description: 'collection.dropIndexes retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -741,7 +1705,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [dropIndexes] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -751,3 +1715,34 @@ tests: object: *collection name: dropIndexes expectError: true + + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + diff --git a/source/client-backpressure/tests/backpressure-retry-max-attempts.yml.template b/source/client-backpressure/tests/backpressure-retry-max-attempts.yml.template index bf089211fd..555b90425d 100644 --- a/source/client-backpressure/tests/backpressure-retry-max-attempts.yml.template +++ b/source/client-backpressure/tests/backpressure-retry-max-attempts.yml.template @@ -14,6 +14,7 @@ createEntities: client: id: &client client useMultipleMongoses: false + observeEvents: [ 'commandStartedEvent', 'commandSucceededEvent', 'commandFailedEvent' ] - client: @@ -43,6 +44,10 @@ tests: {% for operation in operations %} - description: '{{operation.object}}.{{operation.operation_name}} retries at most maxAttempts=5 times' + {%- if ((operation.operation_name == 'clientBulkWrite')) %} + runOnRequirements: + minServerVersion: '8.0' + {%- endif %} operations: - name: failPoint object: testRunner @@ -50,7 +55,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [{{operation.command_name}}] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -69,4 +74,35 @@ tests: saveResultAsEntity: changeStream {%- endif %} expectError: true + + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: {{operation.command_name}} + - commandFailedEvent: + commandName: {{operation.command_name}} + - commandStartedEvent: + commandName: {{operation.command_name}} + - commandFailedEvent: + commandName: {{operation.command_name}} + - commandStartedEvent: + commandName: {{operation.command_name}} + - commandFailedEvent: + commandName: {{operation.command_name}} + - commandStartedEvent: + commandName: {{operation.command_name}} + - commandFailedEvent: + commandName: {{operation.command_name}} + - commandStartedEvent: + commandName: {{operation.command_name}} + - commandFailedEvent: + commandName: {{operation.command_name}} + - commandStartedEvent: + commandName: {{operation.command_name}} + - commandFailedEvent: + commandName: {{operation.command_name}} + {% endfor -%} From 52e2a352f30f4a7e02b4ecd7f7370b38daab6752 Mon Sep 17 00:00:00 2001 From: bailey Date: Tue, 2 Dec 2025 10:12:19 -0700 Subject: [PATCH 06/12] fix run on requirements --- .../tests/backpressure-retry-loop.json | 8 +++++--- .../client-backpressure/tests/backpressure-retry-loop.yml | 2 +- .../tests/backpressure-retry-loop.yml.template | 2 +- .../tests/backpressure-retry-max-attempts.json | 8 +++++--- .../tests/backpressure-retry-max-attempts.yml | 2 +- .../tests/backpressure-retry-max-attempts.yml.template | 2 +- 6 files changed, 14 insertions(+), 10 deletions(-) diff --git a/source/client-backpressure/tests/backpressure-retry-loop.json b/source/client-backpressure/tests/backpressure-retry-loop.json index ae944f73ad..749159d4ae 100644 --- a/source/client-backpressure/tests/backpressure-retry-loop.json +++ b/source/client-backpressure/tests/backpressure-retry-loop.json @@ -381,9 +381,11 @@ }, { "description": "client.clientBulkWrite retries using operation loop", - "runOnRequirements": { - "minServerVersion": "8.0" - }, + "runOnRequirements": [ + { + "minServerVersion": "8.0" + } + ], "operations": [ { "object": "utilCollection", diff --git a/source/client-backpressure/tests/backpressure-retry-loop.yml b/source/client-backpressure/tests/backpressure-retry-loop.yml index c612986233..c069afa6be 100644 --- a/source/client-backpressure/tests/backpressure-retry-loop.yml +++ b/source/client-backpressure/tests/backpressure-retry-loop.yml @@ -224,7 +224,7 @@ tests: - description: 'client.clientBulkWrite retries using operation loop' runOnRequirements: - minServerVersion: '8.0' + - minServerVersion: '8.0' # client bulk write added to server in 8.0 operations: - object: *utilCollection diff --git a/source/client-backpressure/tests/backpressure-retry-loop.yml.template b/source/client-backpressure/tests/backpressure-retry-loop.yml.template index cad83625d3..6101d17178 100644 --- a/source/client-backpressure/tests/backpressure-retry-loop.yml.template +++ b/source/client-backpressure/tests/backpressure-retry-loop.yml.template @@ -58,7 +58,7 @@ tests: description: '{{operation.object}}.{{operation.operation_name}} retries using operation loop' {%- if ((operation.operation_name == 'clientBulkWrite')) %} runOnRequirements: - minServerVersion: '8.0' + - minServerVersion: '8.0' # client bulk write added to server in 8.0 {%- endif %} operations: - diff --git a/source/client-backpressure/tests/backpressure-retry-max-attempts.json b/source/client-backpressure/tests/backpressure-retry-max-attempts.json index 7e9cc67d7a..5cc90248d2 100644 --- a/source/client-backpressure/tests/backpressure-retry-max-attempts.json +++ b/source/client-backpressure/tests/backpressure-retry-max-attempts.json @@ -364,9 +364,11 @@ }, { "description": "client.clientBulkWrite retries at most maxAttempts=5 times", - "runOnRequirements": { - "minServerVersion": "8.0" - }, + "runOnRequirements": [ + { + "minServerVersion": "8.0" + } + ], "operations": [ { "name": "failPoint", diff --git a/source/client-backpressure/tests/backpressure-retry-max-attempts.yml b/source/client-backpressure/tests/backpressure-retry-max-attempts.yml index 9161828e99..9bbbd74a49 100644 --- a/source/client-backpressure/tests/backpressure-retry-max-attempts.yml +++ b/source/client-backpressure/tests/backpressure-retry-max-attempts.yml @@ -203,7 +203,7 @@ tests: - description: 'client.clientBulkWrite retries at most maxAttempts=5 times' runOnRequirements: - minServerVersion: '8.0' + - minServerVersion: '8.0' # client bulk write added to server in 8.0 operations: - name: failPoint object: testRunner diff --git a/source/client-backpressure/tests/backpressure-retry-max-attempts.yml.template b/source/client-backpressure/tests/backpressure-retry-max-attempts.yml.template index 555b90425d..342556a6c3 100644 --- a/source/client-backpressure/tests/backpressure-retry-max-attempts.yml.template +++ b/source/client-backpressure/tests/backpressure-retry-max-attempts.yml.template @@ -46,7 +46,7 @@ tests: description: '{{operation.object}}.{{operation.operation_name}} retries at most maxAttempts=5 times' {%- if ((operation.operation_name == 'clientBulkWrite')) %} runOnRequirements: - minServerVersion: '8.0' + - minServerVersion: '8.0' # client bulk write added to server in 8.0 {%- endif %} operations: - name: failPoint From 391c95190bb0d1aa286ff2bd863258e43bcd0eef Mon Sep 17 00:00:00 2001 From: bailey Date: Tue, 2 Dec 2025 10:29:33 -0700 Subject: [PATCH 07/12] fix run on requirements? --- source/client-backpressure/tests/backpressure-retry-loop.json | 2 +- source/client-backpressure/tests/backpressure-retry-loop.yml | 2 +- .../tests/backpressure-retry-loop.yml.template | 2 +- .../tests/backpressure-retry-max-attempts.json | 2 +- .../tests/backpressure-retry-max-attempts.yml | 2 +- .../tests/backpressure-retry-max-attempts.yml.template | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/source/client-backpressure/tests/backpressure-retry-loop.json b/source/client-backpressure/tests/backpressure-retry-loop.json index 749159d4ae..63b3bd5e91 100644 --- a/source/client-backpressure/tests/backpressure-retry-loop.json +++ b/source/client-backpressure/tests/backpressure-retry-loop.json @@ -1,6 +1,6 @@ { "description": "tests that operations respect overload backoff retry loop", - "schemaVersion": "1.0", + "schemaVersion": "1.3", "runOnRequirements": [ { "minServerVersion": "4.4", diff --git a/source/client-backpressure/tests/backpressure-retry-loop.yml b/source/client-backpressure/tests/backpressure-retry-loop.yml index c069afa6be..8a0ea01766 100644 --- a/source/client-backpressure/tests/backpressure-retry-loop.yml +++ b/source/client-backpressure/tests/backpressure-retry-loop.yml @@ -2,7 +2,7 @@ description: tests that operations respect overload backoff retry loop -schemaVersion: '1.0' +schemaVersion: '1.3' runOnRequirements: - diff --git a/source/client-backpressure/tests/backpressure-retry-loop.yml.template b/source/client-backpressure/tests/backpressure-retry-loop.yml.template index 6101d17178..fca19f94f7 100644 --- a/source/client-backpressure/tests/backpressure-retry-loop.yml.template +++ b/source/client-backpressure/tests/backpressure-retry-loop.yml.template @@ -2,7 +2,7 @@ description: tests that operations respect overload backoff retry loop -schemaVersion: '1.0' +schemaVersion: '1.3' runOnRequirements: - diff --git a/source/client-backpressure/tests/backpressure-retry-max-attempts.json b/source/client-backpressure/tests/backpressure-retry-max-attempts.json index 5cc90248d2..206c86fe51 100644 --- a/source/client-backpressure/tests/backpressure-retry-max-attempts.json +++ b/source/client-backpressure/tests/backpressure-retry-max-attempts.json @@ -1,6 +1,6 @@ { "description": "tests that operations retry at most maxAttempts=5 times", - "schemaVersion": "1.0", + "schemaVersion": "1.3", "runOnRequirements": [ { "minServerVersion": "4.4", diff --git a/source/client-backpressure/tests/backpressure-retry-max-attempts.yml b/source/client-backpressure/tests/backpressure-retry-max-attempts.yml index 9bbbd74a49..edcac5eb87 100644 --- a/source/client-backpressure/tests/backpressure-retry-max-attempts.yml +++ b/source/client-backpressure/tests/backpressure-retry-max-attempts.yml @@ -2,7 +2,7 @@ description: tests that operations retry at most maxAttempts=5 times -schemaVersion: '1.0' +schemaVersion: '1.3' runOnRequirements: - diff --git a/source/client-backpressure/tests/backpressure-retry-max-attempts.yml.template b/source/client-backpressure/tests/backpressure-retry-max-attempts.yml.template index 342556a6c3..784fc0bb2a 100644 --- a/source/client-backpressure/tests/backpressure-retry-max-attempts.yml.template +++ b/source/client-backpressure/tests/backpressure-retry-max-attempts.yml.template @@ -2,7 +2,7 @@ description: tests that operations retry at most maxAttempts=5 times -schemaVersion: '1.0' +schemaVersion: '1.3' runOnRequirements: - From 92501c0f0e95c2c0f4d49ac9a6a8548a20a2f4f5 Mon Sep 17 00:00:00 2001 From: bailey Date: Tue, 2 Dec 2025 10:42:09 -0700 Subject: [PATCH 08/12] fix CI --- source/client-backpressure/client-backpressure.md | 4 ++++ source/logging/logging.md | 2 +- source/retryable-writes/retryable-writes.md | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/source/client-backpressure/client-backpressure.md b/source/client-backpressure/client-backpressure.md index 88dd94807e..c2bcecc545 100644 --- a/source/client-backpressure/client-backpressure.md +++ b/source/client-backpressure/client-backpressure.md @@ -390,6 +390,10 @@ attempt. Note that the second `CommandStartedEvent` and "command started" log me application from inadvertently retrying for too long (see [Backwards Compatibility](#backwards-compatibility) for details). +### Backwards Compatibility + +TODO + ## Test Plan See the [README](./tests/README.md) for tests. diff --git a/source/logging/logging.md b/source/logging/logging.md index 6f7c36c200..8971dbcde7 100644 --- a/source/logging/logging.md +++ b/source/logging/logging.md @@ -95,7 +95,7 @@ Drivers MUST support configuring where log messages should be output, including > - If the value is "stdout" (case-insensitive), log to stdout. > - If the value is "stderr" (case-insensitive), log to stderr. > - Else, if direct logging to files is supported, log to a file at the specified path. If the file already exists, it - > MUST be appended to. + > MUST be appended to. > > If the variable is not provided or is set to an invalid value (which could be invalid for any reason, e.g. the path > does not exist or is not writeable), the driver MUST log to stderr and the driver MAY attempt to warn the user about diff --git a/source/retryable-writes/retryable-writes.md b/source/retryable-writes/retryable-writes.md index fa613a2908..4a614be89a 100644 --- a/source/retryable-writes/retryable-writes.md +++ b/source/retryable-writes/retryable-writes.md @@ -50,7 +50,7 @@ across drivers. **Retryable Write Error** An error is considered retryable if it has a RetryableWriteError label in its top-level "errorLabels" field. See -[Determining Retryable Write Errors](#determining-retryable-errors) for more information. +[Determining Retryable Write Errors](#determining-retryable-write-errors) for more information. Additional terms may be defined in the [Driver Session](../sessions/driver-sessions.md) specification. From 0fdef39ac06f663ee2f38c41f71c609e856016b5 Mon Sep 17 00:00:00 2001 From: bailey Date: Wed, 3 Dec 2025 11:36:46 -0700 Subject: [PATCH 09/12] comments --- source/client-backpressure/client-backpressure.md | 7 +++++++ source/retryable-reads/retryable-reads.md | 7 ++++--- source/retryable-writes/retryable-writes.md | 7 ++++--- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/source/client-backpressure/client-backpressure.md b/source/client-backpressure/client-backpressure.md index c2bcecc545..39613c224d 100644 --- a/source/client-backpressure/client-backpressure.md +++ b/source/client-backpressure/client-backpressure.md @@ -67,6 +67,13 @@ closed"). When a new connection attempt is queued by the server for so long that the driver-side timeout expires, drivers will observe this as a network timeout error. +#### Goodput + +The throughput of positive, useful output. In the context of drivers, this refers to the number of non-error results +that the driver processes per unit of time. + +See [goodput](https://en.wikipedia.org/wiki/Goodput). + ### Requirements for Client Backpressure #### Overload retry policy diff --git a/source/retryable-reads/retryable-reads.md b/source/retryable-reads/retryable-reads.md index a7100cbe92..42046e8b9a 100644 --- a/source/retryable-reads/retryable-reads.md +++ b/source/retryable-reads/retryable-reads.md @@ -15,9 +15,10 @@ This specification will - outline how an API for retryable read operations will be implemented in drivers - define an option to enable retryable reads for an application. -The changes in this specification are related to but distinct from the retryability behaviors defined in the client -backpressure specification, which defines a retryability mechanism for all commands under certain server conditions. -Unless otherwise noted, the changes in this specification refer only to the retryability behaviors summarized above. +The changes in this specification are related to but distinct from the retryability behaviors defined in the +[Client Backpressure Specification](../client-backpressure/client-backpressure.md), which defines a retryability +mechanism for all commands under certain server conditions. Unless otherwise noted, the changes in this specification +refer only to the retryability behaviors summarized above. ## META diff --git a/source/retryable-writes/retryable-writes.md b/source/retryable-writes/retryable-writes.md index 4a614be89a..d8cf74c009 100644 --- a/source/retryable-writes/retryable-writes.md +++ b/source/retryable-writes/retryable-writes.md @@ -19,9 +19,10 @@ specification will outline how an API for retryable write operations will be imp will define an option to enable retryable writes for an application and describe how a transaction ID will be provided to write commands executed therein. -The changes in this specification are related to but distinct from the retryability behaviors defined in the client -backpressure specification, which defines a retryability mechanism for all commands under certain server conditions. -Unless otherwise noted, the changes in this specification refer only to the retryability behaviors summarized above. +The changes in this specification are related to but distinct from the retryability behaviors defined in the +[Client Backpressure Specification](../client-backpressure/client-backpressure.md), which defines a retryability +mechanism for all commands under certain server conditions. Unless otherwise noted, the changes in this specification +refer only to the retryability behaviors summarized above. ## META From 82acab8e2238497392961c1b210ab5f0d0fa86bb Mon Sep 17 00:00:00 2001 From: bailey Date: Wed, 3 Dec 2025 11:47:35 -0700 Subject: [PATCH 10/12] Fix broken unified tests --- .../tests/backpressure-retry-loop.json | 96 +++++-------- .../tests/backpressure-retry-loop.yml | 32 ----- .../backpressure-retry-loop.yml.template | 1 - .../backpressure-retry-max-attempts.json | 128 +++++++++++++----- .../tests/backpressure-retry-max-attempts.yml | 96 ++++++++----- ...ckpressure-retry-max-attempts.yml.template | 3 +- 6 files changed, 194 insertions(+), 162 deletions(-) diff --git a/source/client-backpressure/tests/backpressure-retry-loop.json b/source/client-backpressure/tests/backpressure-retry-loop.json index 63b3bd5e91..b121d9bd3d 100644 --- a/source/client-backpressure/tests/backpressure-retry-loop.json +++ b/source/client-backpressure/tests/backpressure-retry-loop.json @@ -125,8 +125,7 @@ "name": "listDatabases", "arguments": { "filter": {} - }, - "expectError": false + } } ], "expectEvents": [ @@ -224,8 +223,7 @@ }, { "object": "client", - "name": "listDatabaseNames", - "expectError": false + "name": "listDatabaseNames" } ], "expectEvents": [ @@ -327,8 +325,7 @@ "arguments": { "pipeline": [] }, - "saveResultAsEntity": "changeStream", - "expectError": false + "saveResultAsEntity": "changeStream" } ], "expectEvents": [ @@ -444,8 +441,7 @@ } } ] - }, - "expectError": false + } } ], "expectEvents": [ @@ -553,8 +549,7 @@ "$limit": 1 } ] - }, - "expectError": false + } } ], "expectEvents": [ @@ -655,8 +650,7 @@ "name": "listCollections", "arguments": { "filter": {} - }, - "expectError": false + } } ], "expectEvents": [ @@ -757,8 +751,7 @@ "name": "listCollectionNames", "arguments": { "filter": {} - }, - "expectError": false + } } ], "expectEvents": [ @@ -862,8 +855,7 @@ "ping": 1 }, "commandName": "ping" - }, - "expectError": false + } } ], "expectEvents": [ @@ -965,8 +957,7 @@ "arguments": { "pipeline": [] }, - "saveResultAsEntity": "changeStream", - "expectError": false + "saveResultAsEntity": "changeStream" } ], "expectEvents": [ @@ -1067,8 +1058,7 @@ "name": "aggregate", "arguments": { "pipeline": [] - }, - "expectError": false + } } ], "expectEvents": [ @@ -1169,8 +1159,7 @@ "name": "countDocuments", "arguments": { "filter": {} - }, - "expectError": false + } } ], "expectEvents": [ @@ -1268,8 +1257,7 @@ }, { "object": "collection", - "name": "estimatedDocumentCount", - "expectError": false + "name": "estimatedDocumentCount" } ], "expectEvents": [ @@ -1371,8 +1359,7 @@ "arguments": { "fieldName": "x", "filter": {} - }, - "expectError": false + } } ], "expectEvents": [ @@ -1473,8 +1460,7 @@ "name": "find", "arguments": { "filter": {} - }, - "expectError": false + } } ], "expectEvents": [ @@ -1575,8 +1561,7 @@ "name": "findOne", "arguments": { "filter": {} - }, - "expectError": false + } } ], "expectEvents": [ @@ -1674,8 +1659,7 @@ }, { "object": "collection", - "name": "listIndexes", - "expectError": false + "name": "listIndexes" } ], "expectEvents": [ @@ -1773,8 +1757,7 @@ }, { "object": "collection", - "name": "listIndexNames", - "expectError": false + "name": "listIndexNames" } ], "expectEvents": [ @@ -1876,8 +1859,7 @@ "arguments": { "pipeline": [] }, - "saveResultAsEntity": "changeStream", - "expectError": false + "saveResultAsEntity": "changeStream" } ], "expectEvents": [ @@ -1981,8 +1963,7 @@ "_id": 2, "x": 22 } - }, - "expectError": false + } } ], "expectEvents": [ @@ -2088,8 +2069,7 @@ "x": 22 } ] - }, - "expectError": false + } } ], "expectEvents": [ @@ -2190,8 +2170,7 @@ "name": "deleteOne", "arguments": { "filter": {} - }, - "expectError": false + } } ], "expectEvents": [ @@ -2292,8 +2271,7 @@ "name": "deleteMany", "arguments": { "filter": {} - }, - "expectError": false + } } ], "expectEvents": [ @@ -2397,8 +2375,7 @@ "replacement": { "x": 22 } - }, - "expectError": false + } } ], "expectEvents": [ @@ -2504,8 +2481,7 @@ "x": 22 } } - }, - "expectError": false + } } ], "expectEvents": [ @@ -2611,8 +2587,7 @@ "x": 22 } } - }, - "expectError": false + } } ], "expectEvents": [ @@ -2713,8 +2688,7 @@ "name": "findOneAndDelete", "arguments": { "filter": {} - }, - "expectError": false + } } ], "expectEvents": [ @@ -2818,8 +2792,7 @@ "replacement": { "x": 22 } - }, - "expectError": false + } } ], "expectEvents": [ @@ -2925,8 +2898,7 @@ "x": 22 } } - }, - "expectError": false + } } ], "expectEvents": [ @@ -3036,8 +3008,7 @@ } } ] - }, - "expectError": false + } } ], "expectEvents": [ @@ -3141,8 +3112,7 @@ "x": 11 }, "name": "x_11" - }, - "expectError": false + } } ], "expectEvents": [ @@ -3253,8 +3223,7 @@ "name": "dropIndex", "arguments": { "name": "x_11" - }, - "expectError": false + } } ], "expectEvents": [ @@ -3352,8 +3321,7 @@ }, { "object": "collection", - "name": "dropIndexes", - "expectError": false + "name": "dropIndexes" } ], "expectEvents": [ diff --git a/source/client-backpressure/tests/backpressure-retry-loop.yml b/source/client-backpressure/tests/backpressure-retry-loop.yml index 8a0ea01766..08d6072edd 100644 --- a/source/client-backpressure/tests/backpressure-retry-loop.yml +++ b/source/client-backpressure/tests/backpressure-retry-loop.yml @@ -88,7 +88,6 @@ tests: name: listDatabases arguments: filter: {} - expectError: false expectEvents: - client: "client" @@ -142,7 +141,6 @@ tests: - object: *client name: listDatabaseNames - expectError: false expectEvents: - client: "client" @@ -199,7 +197,6 @@ tests: arguments: pipeline: [] saveResultAsEntity: changeStream - expectError: false expectEvents: - client: "client" @@ -260,7 +257,6 @@ tests: - insertOne: namespace: retryable-writes-tests.coll document: { _id: 8, x: 88 } - expectError: false expectEvents: - client: "client" @@ -316,7 +312,6 @@ tests: name: aggregate arguments: pipeline: [ { $listLocalSessions: {} }, { $limit: 1 } ] - expectError: false expectEvents: - client: "client" @@ -372,7 +367,6 @@ tests: name: listCollections arguments: filter: {} - expectError: false expectEvents: - client: "client" @@ -428,7 +422,6 @@ tests: name: listCollectionNames arguments: filter: {} - expectError: false expectEvents: - client: "client" @@ -485,7 +478,6 @@ tests: arguments: command: { ping: 1 } commandName: ping - expectError: false expectEvents: - client: "client" @@ -542,7 +534,6 @@ tests: arguments: pipeline: [] saveResultAsEntity: changeStream - expectError: false expectEvents: - client: "client" @@ -598,7 +589,6 @@ tests: name: aggregate arguments: pipeline: [] - expectError: false expectEvents: - client: "client" @@ -654,7 +644,6 @@ tests: name: countDocuments arguments: filter: {} - expectError: false expectEvents: - client: "client" @@ -708,7 +697,6 @@ tests: - object: *collection name: estimatedDocumentCount - expectError: false expectEvents: - client: "client" @@ -765,7 +753,6 @@ tests: arguments: fieldName: x filter: {} - expectError: false expectEvents: - client: "client" @@ -821,7 +808,6 @@ tests: name: find arguments: filter: {} - expectError: false expectEvents: - client: "client" @@ -877,7 +863,6 @@ tests: name: findOne arguments: filter: {} - expectError: false expectEvents: - client: "client" @@ -931,7 +916,6 @@ tests: - object: *collection name: listIndexes - expectError: false expectEvents: - client: "client" @@ -985,7 +969,6 @@ tests: - object: *collection name: listIndexNames - expectError: false expectEvents: - client: "client" @@ -1042,7 +1025,6 @@ tests: arguments: pipeline: [] saveResultAsEntity: changeStream - expectError: false expectEvents: - client: "client" @@ -1098,7 +1080,6 @@ tests: name: insertOne arguments: document: { _id: 2, x: 22 } - expectError: false expectEvents: - client: "client" @@ -1155,7 +1136,6 @@ tests: arguments: documents: - { _id: 2, x: 22 } - expectError: false expectEvents: - client: "client" @@ -1211,7 +1191,6 @@ tests: name: deleteOne arguments: filter: {} - expectError: false expectEvents: - client: "client" @@ -1267,7 +1246,6 @@ tests: name: deleteMany arguments: filter: {} - expectError: false expectEvents: - client: "client" @@ -1324,7 +1302,6 @@ tests: arguments: filter: {} replacement: { x: 22 } - expectError: false expectEvents: - client: "client" @@ -1381,7 +1358,6 @@ tests: arguments: filter: {} update: { $set: { x: 22 } } - expectError: false expectEvents: - client: "client" @@ -1438,7 +1414,6 @@ tests: arguments: filter: {} update: { $set: { x: 22 } } - expectError: false expectEvents: - client: "client" @@ -1494,7 +1469,6 @@ tests: name: findOneAndDelete arguments: filter: {} - expectError: false expectEvents: - client: "client" @@ -1551,7 +1525,6 @@ tests: arguments: filter: {} replacement: { x: 22 } - expectError: false expectEvents: - client: "client" @@ -1608,7 +1581,6 @@ tests: arguments: filter: {} update: { $set: { x: 22 } } - expectError: false expectEvents: - client: "client" @@ -1666,7 +1638,6 @@ tests: requests: - insertOne: document: { _id: 2, x: 22 } - expectError: false expectEvents: - client: "client" @@ -1723,7 +1694,6 @@ tests: arguments: keys: { x: 11 } name: "x_11" - expectError: false expectEvents: - client: "client" @@ -1785,7 +1755,6 @@ tests: name: dropIndex arguments: name: "x_11" - expectError: false expectEvents: - client: "client" @@ -1839,7 +1808,6 @@ tests: - object: *collection name: dropIndexes - expectError: false expectEvents: - client: "client" diff --git a/source/client-backpressure/tests/backpressure-retry-loop.yml.template b/source/client-backpressure/tests/backpressure-retry-loop.yml.template index fca19f94f7..db3ce498e9 100644 --- a/source/client-backpressure/tests/backpressure-retry-loop.yml.template +++ b/source/client-backpressure/tests/backpressure-retry-loop.yml.template @@ -108,7 +108,6 @@ tests: {%- if operation.operation_name == "createChangeStream" %} saveResultAsEntity: changeStream {%- endif %} - expectError: false expectEvents: - client: "client" diff --git a/source/client-backpressure/tests/backpressure-retry-max-attempts.json b/source/client-backpressure/tests/backpressure-retry-max-attempts.json index 206c86fe51..d11ad9c5c5 100644 --- a/source/client-backpressure/tests/backpressure-retry-max-attempts.json +++ b/source/client-backpressure/tests/backpressure-retry-max-attempts.json @@ -91,7 +91,9 @@ "arguments": { "filter": {} }, - "expectError": true + "expectError": { + "isError": true + } } ], "expectEvents": [ @@ -189,7 +191,9 @@ { "object": "client", "name": "listDatabaseNames", - "expectError": true + "expectError": { + "isError": true + } } ], "expectEvents": [ @@ -291,7 +295,9 @@ "pipeline": [] }, "saveResultAsEntity": "changeStream", - "expectError": true + "expectError": { + "isError": true + } } ], "expectEvents": [ @@ -407,7 +413,9 @@ } ] }, - "expectError": true + "expectError": { + "isError": true + } } ], "expectEvents": [ @@ -515,7 +523,9 @@ } ] }, - "expectError": true + "expectError": { + "isError": true + } } ], "expectEvents": [ @@ -616,7 +626,9 @@ "arguments": { "filter": {} }, - "expectError": true + "expectError": { + "isError": true + } } ], "expectEvents": [ @@ -717,7 +729,9 @@ "arguments": { "filter": {} }, - "expectError": true + "expectError": { + "isError": true + } } ], "expectEvents": [ @@ -821,7 +835,9 @@ }, "commandName": "ping" }, - "expectError": true + "expectError": { + "isError": true + } } ], "expectEvents": [ @@ -923,7 +939,9 @@ "pipeline": [] }, "saveResultAsEntity": "changeStream", - "expectError": true + "expectError": { + "isError": true + } } ], "expectEvents": [ @@ -1024,7 +1042,9 @@ "arguments": { "pipeline": [] }, - "expectError": true + "expectError": { + "isError": true + } } ], "expectEvents": [ @@ -1125,7 +1145,9 @@ "arguments": { "filter": {} }, - "expectError": true + "expectError": { + "isError": true + } } ], "expectEvents": [ @@ -1223,7 +1245,9 @@ { "object": "collection", "name": "estimatedDocumentCount", - "expectError": true + "expectError": { + "isError": true + } } ], "expectEvents": [ @@ -1325,7 +1349,9 @@ "fieldName": "x", "filter": {} }, - "expectError": true + "expectError": { + "isError": true + } } ], "expectEvents": [ @@ -1426,7 +1452,9 @@ "arguments": { "filter": {} }, - "expectError": true + "expectError": { + "isError": true + } } ], "expectEvents": [ @@ -1527,7 +1555,9 @@ "arguments": { "filter": {} }, - "expectError": true + "expectError": { + "isError": true + } } ], "expectEvents": [ @@ -1625,7 +1655,9 @@ { "object": "collection", "name": "listIndexes", - "expectError": true + "expectError": { + "isError": true + } } ], "expectEvents": [ @@ -1723,7 +1755,9 @@ { "object": "collection", "name": "listIndexNames", - "expectError": true + "expectError": { + "isError": true + } } ], "expectEvents": [ @@ -1825,7 +1859,9 @@ "pipeline": [] }, "saveResultAsEntity": "changeStream", - "expectError": true + "expectError": { + "isError": true + } } ], "expectEvents": [ @@ -1929,7 +1965,9 @@ "x": 22 } }, - "expectError": true + "expectError": { + "isError": true + } } ], "expectEvents": [ @@ -2035,7 +2073,9 @@ } ] }, - "expectError": true + "expectError": { + "isError": true + } } ], "expectEvents": [ @@ -2136,7 +2176,9 @@ "arguments": { "filter": {} }, - "expectError": true + "expectError": { + "isError": true + } } ], "expectEvents": [ @@ -2237,7 +2279,9 @@ "arguments": { "filter": {} }, - "expectError": true + "expectError": { + "isError": true + } } ], "expectEvents": [ @@ -2341,7 +2385,9 @@ "x": 22 } }, - "expectError": true + "expectError": { + "isError": true + } } ], "expectEvents": [ @@ -2447,7 +2493,9 @@ } } }, - "expectError": true + "expectError": { + "isError": true + } } ], "expectEvents": [ @@ -2553,7 +2601,9 @@ } } }, - "expectError": true + "expectError": { + "isError": true + } } ], "expectEvents": [ @@ -2654,7 +2704,9 @@ "arguments": { "filter": {} }, - "expectError": true + "expectError": { + "isError": true + } } ], "expectEvents": [ @@ -2758,7 +2810,9 @@ "x": 22 } }, - "expectError": true + "expectError": { + "isError": true + } } ], "expectEvents": [ @@ -2864,7 +2918,9 @@ } } }, - "expectError": true + "expectError": { + "isError": true + } } ], "expectEvents": [ @@ -2974,7 +3030,9 @@ } ] }, - "expectError": true + "expectError": { + "isError": true + } } ], "expectEvents": [ @@ -3078,7 +3136,9 @@ }, "name": "x_11" }, - "expectError": true + "expectError": { + "isError": true + } } ], "expectEvents": [ @@ -3179,7 +3239,9 @@ "arguments": { "name": "x_11" }, - "expectError": true + "expectError": { + "isError": true + } } ], "expectEvents": [ @@ -3277,7 +3339,9 @@ { "object": "collection", "name": "dropIndexes", - "expectError": true + "expectError": { + "isError": true + } } ], "expectEvents": [ diff --git a/source/client-backpressure/tests/backpressure-retry-max-attempts.yml b/source/client-backpressure/tests/backpressure-retry-max-attempts.yml index edcac5eb87..72c9b446b7 100644 --- a/source/client-backpressure/tests/backpressure-retry-max-attempts.yml +++ b/source/client-backpressure/tests/backpressure-retry-max-attempts.yml @@ -62,7 +62,8 @@ tests: name: listDatabases arguments: filter: {} - expectError: true + expectError: + isError: true expectEvents: - client: "client" @@ -113,7 +114,8 @@ tests: - object: *client name: listDatabaseNames - expectError: true + expectError: + isError: true expectEvents: - client: "client" @@ -167,7 +169,8 @@ tests: arguments: pipeline: [] saveResultAsEntity: changeStream - expectError: true + expectError: + isError: true expectEvents: - client: "client" @@ -225,7 +228,8 @@ tests: - insertOne: namespace: retryable-writes-tests.coll document: { _id: 8, x: 88 } - expectError: true + expectError: + isError: true expectEvents: - client: "client" @@ -278,7 +282,8 @@ tests: name: aggregate arguments: pipeline: [ { $listLocalSessions: {} }, { $limit: 1 } ] - expectError: true + expectError: + isError: true expectEvents: - client: "client" @@ -331,7 +336,8 @@ tests: name: listCollections arguments: filter: {} - expectError: true + expectError: + isError: true expectEvents: - client: "client" @@ -384,7 +390,8 @@ tests: name: listCollectionNames arguments: filter: {} - expectError: true + expectError: + isError: true expectEvents: - client: "client" @@ -438,7 +445,8 @@ tests: arguments: command: { ping: 1 } commandName: ping - expectError: true + expectError: + isError: true expectEvents: - client: "client" @@ -492,7 +500,8 @@ tests: arguments: pipeline: [] saveResultAsEntity: changeStream - expectError: true + expectError: + isError: true expectEvents: - client: "client" @@ -545,7 +554,8 @@ tests: name: aggregate arguments: pipeline: [] - expectError: true + expectError: + isError: true expectEvents: - client: "client" @@ -598,7 +608,8 @@ tests: name: countDocuments arguments: filter: {} - expectError: true + expectError: + isError: true expectEvents: - client: "client" @@ -649,7 +660,8 @@ tests: - object: *collection name: estimatedDocumentCount - expectError: true + expectError: + isError: true expectEvents: - client: "client" @@ -703,7 +715,8 @@ tests: arguments: fieldName: x filter: {} - expectError: true + expectError: + isError: true expectEvents: - client: "client" @@ -756,7 +769,8 @@ tests: name: find arguments: filter: {} - expectError: true + expectError: + isError: true expectEvents: - client: "client" @@ -809,7 +823,8 @@ tests: name: findOne arguments: filter: {} - expectError: true + expectError: + isError: true expectEvents: - client: "client" @@ -860,7 +875,8 @@ tests: - object: *collection name: listIndexes - expectError: true + expectError: + isError: true expectEvents: - client: "client" @@ -911,7 +927,8 @@ tests: - object: *collection name: listIndexNames - expectError: true + expectError: + isError: true expectEvents: - client: "client" @@ -965,7 +982,8 @@ tests: arguments: pipeline: [] saveResultAsEntity: changeStream - expectError: true + expectError: + isError: true expectEvents: - client: "client" @@ -1018,7 +1036,8 @@ tests: name: insertOne arguments: document: { _id: 2, x: 22 } - expectError: true + expectError: + isError: true expectEvents: - client: "client" @@ -1072,7 +1091,8 @@ tests: arguments: documents: - { _id: 2, x: 22 } - expectError: true + expectError: + isError: true expectEvents: - client: "client" @@ -1125,7 +1145,8 @@ tests: name: deleteOne arguments: filter: {} - expectError: true + expectError: + isError: true expectEvents: - client: "client" @@ -1178,7 +1199,8 @@ tests: name: deleteMany arguments: filter: {} - expectError: true + expectError: + isError: true expectEvents: - client: "client" @@ -1232,7 +1254,8 @@ tests: arguments: filter: {} replacement: { x: 22 } - expectError: true + expectError: + isError: true expectEvents: - client: "client" @@ -1286,7 +1309,8 @@ tests: arguments: filter: {} update: { $set: { x: 22 } } - expectError: true + expectError: + isError: true expectEvents: - client: "client" @@ -1340,7 +1364,8 @@ tests: arguments: filter: {} update: { $set: { x: 22 } } - expectError: true + expectError: + isError: true expectEvents: - client: "client" @@ -1393,7 +1418,8 @@ tests: name: findOneAndDelete arguments: filter: {} - expectError: true + expectError: + isError: true expectEvents: - client: "client" @@ -1447,7 +1473,8 @@ tests: arguments: filter: {} replacement: { x: 22 } - expectError: true + expectError: + isError: true expectEvents: - client: "client" @@ -1501,7 +1528,8 @@ tests: arguments: filter: {} update: { $set: { x: 22 } } - expectError: true + expectError: + isError: true expectEvents: - client: "client" @@ -1556,7 +1584,8 @@ tests: requests: - insertOne: document: { _id: 2, x: 22 } - expectError: true + expectError: + isError: true expectEvents: - client: "client" @@ -1610,7 +1639,8 @@ tests: arguments: keys: { x: 11 } name: "x_11" - expectError: true + expectError: + isError: true expectEvents: - client: "client" @@ -1663,7 +1693,8 @@ tests: name: dropIndex arguments: name: "x_11" - expectError: true + expectError: + isError: true expectEvents: - client: "client" @@ -1714,7 +1745,8 @@ tests: - object: *collection name: dropIndexes - expectError: true + expectError: + isError: true expectEvents: - client: "client" diff --git a/source/client-backpressure/tests/backpressure-retry-max-attempts.yml.template b/source/client-backpressure/tests/backpressure-retry-max-attempts.yml.template index 784fc0bb2a..2920a4064f 100644 --- a/source/client-backpressure/tests/backpressure-retry-max-attempts.yml.template +++ b/source/client-backpressure/tests/backpressure-retry-max-attempts.yml.template @@ -73,7 +73,8 @@ tests: {%- if operation.operation_name == "createChangeStream" %} saveResultAsEntity: changeStream {%- endif %} - expectError: true + expectError: + isError: true expectEvents: - client: "client" From b3a7b6c77daa69b01667a8a13cf44cf8e11b41d0 Mon Sep 17 00:00:00 2001 From: bailey Date: Wed, 3 Dec 2025 11:58:29 -0700 Subject: [PATCH 11/12] fix UTR linting failures --- .../tests/backpressure-retry-max-attempts.json | 3 --- .../tests/backpressure-retry-max-attempts.yml | 3 --- .../tests/backpressure-retry-max-attempts.yml.template | 3 --- 3 files changed, 9 deletions(-) diff --git a/source/client-backpressure/tests/backpressure-retry-max-attempts.json b/source/client-backpressure/tests/backpressure-retry-max-attempts.json index d11ad9c5c5..a499aa490b 100644 --- a/source/client-backpressure/tests/backpressure-retry-max-attempts.json +++ b/source/client-backpressure/tests/backpressure-retry-max-attempts.json @@ -294,7 +294,6 @@ "arguments": { "pipeline": [] }, - "saveResultAsEntity": "changeStream", "expectError": { "isError": true } @@ -938,7 +937,6 @@ "arguments": { "pipeline": [] }, - "saveResultAsEntity": "changeStream", "expectError": { "isError": true } @@ -1858,7 +1856,6 @@ "arguments": { "pipeline": [] }, - "saveResultAsEntity": "changeStream", "expectError": { "isError": true } diff --git a/source/client-backpressure/tests/backpressure-retry-max-attempts.yml b/source/client-backpressure/tests/backpressure-retry-max-attempts.yml index 72c9b446b7..3bd4582757 100644 --- a/source/client-backpressure/tests/backpressure-retry-max-attempts.yml +++ b/source/client-backpressure/tests/backpressure-retry-max-attempts.yml @@ -168,7 +168,6 @@ tests: name: createChangeStream arguments: pipeline: [] - saveResultAsEntity: changeStream expectError: isError: true @@ -499,7 +498,6 @@ tests: name: createChangeStream arguments: pipeline: [] - saveResultAsEntity: changeStream expectError: isError: true @@ -981,7 +979,6 @@ tests: name: createChangeStream arguments: pipeline: [] - saveResultAsEntity: changeStream expectError: isError: true diff --git a/source/client-backpressure/tests/backpressure-retry-max-attempts.yml.template b/source/client-backpressure/tests/backpressure-retry-max-attempts.yml.template index 2920a4064f..4f2cfeee47 100644 --- a/source/client-backpressure/tests/backpressure-retry-max-attempts.yml.template +++ b/source/client-backpressure/tests/backpressure-retry-max-attempts.yml.template @@ -70,9 +70,6 @@ tests: {{arg}} {%- endfor -%} {%- endif %} - {%- if operation.operation_name == "createChangeStream" %} - saveResultAsEntity: changeStream - {%- endif %} expectError: isError: true From 60a87b8b2f0d315b822e55c66ed84d413c5fe51d Mon Sep 17 00:00:00 2001 From: bailey Date: Wed, 3 Dec 2025 15:22:28 -0700 Subject: [PATCH 12/12] remove broken deleteMany() from unified tests --- .../tests/backpressure-retry-loop.json | 384 ------------------ .../tests/backpressure-retry-loop.yml | 286 ++----------- .../backpressure-retry-loop.yml.template | 7 - 3 files changed, 31 insertions(+), 646 deletions(-) diff --git a/source/client-backpressure/tests/backpressure-retry-loop.json b/source/client-backpressure/tests/backpressure-retry-loop.json index b121d9bd3d..20bdfe3a69 100644 --- a/source/client-backpressure/tests/backpressure-retry-loop.json +++ b/source/client-backpressure/tests/backpressure-retry-loop.json @@ -85,18 +85,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -186,18 +174,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -284,18 +260,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -391,18 +355,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -502,18 +454,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -610,18 +550,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -711,18 +639,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -812,18 +728,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -916,18 +820,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -1018,18 +910,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -1119,18 +999,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -1220,18 +1088,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -1318,18 +1174,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -1420,18 +1264,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -1521,18 +1353,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -1622,18 +1442,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -1720,18 +1528,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -1818,18 +1614,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -1920,18 +1704,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -2024,18 +1796,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -2130,18 +1890,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -2231,18 +1979,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -2332,18 +2068,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -2436,18 +2160,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -2542,18 +2254,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -2648,18 +2348,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -2749,18 +2437,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -2853,18 +2529,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -2959,18 +2623,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -3069,18 +2721,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -3173,18 +2813,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "object": "utilCollection", "name": "createIndex", @@ -3284,18 +2912,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", diff --git a/source/client-backpressure/tests/backpressure-retry-loop.yml b/source/client-backpressure/tests/backpressure-retry-loop.yml index 08d6072edd..a6498568f1 100644 --- a/source/client-backpressure/tests/backpressure-retry-loop.yml +++ b/source/client-backpressure/tests/backpressure-retry-loop.yml @@ -61,14 +61,7 @@ tests: object: *utilCollection name: deleteMany arguments: - filter: {} - - - - object: *utilCollection - name: deleteMany - arguments: - documents: - - { _id: 1, x: 11 } + filter: {} - name: failPoint @@ -116,14 +109,7 @@ tests: object: *utilCollection name: deleteMany arguments: - filter: {} - - - - object: *utilCollection - name: deleteMany - arguments: - documents: - - { _id: 1, x: 11 } + filter: {} - name: failPoint @@ -169,14 +155,7 @@ tests: object: *utilCollection name: deleteMany arguments: - filter: {} - - - - object: *utilCollection - name: deleteMany - arguments: - documents: - - { _id: 1, x: 11 } + filter: {} - name: failPoint @@ -227,14 +206,7 @@ tests: object: *utilCollection name: deleteMany arguments: - filter: {} - - - - object: *utilCollection - name: deleteMany - arguments: - documents: - - { _id: 1, x: 11 } + filter: {} - name: failPoint @@ -285,14 +257,7 @@ tests: object: *utilCollection name: deleteMany arguments: - filter: {} - - - - object: *utilCollection - name: deleteMany - arguments: - documents: - - { _id: 1, x: 11 } + filter: {} - name: failPoint @@ -340,14 +305,7 @@ tests: object: *utilCollection name: deleteMany arguments: - filter: {} - - - - object: *utilCollection - name: deleteMany - arguments: - documents: - - { _id: 1, x: 11 } + filter: {} - name: failPoint @@ -395,14 +353,7 @@ tests: object: *utilCollection name: deleteMany arguments: - filter: {} - - - - object: *utilCollection - name: deleteMany - arguments: - documents: - - { _id: 1, x: 11 } + filter: {} - name: failPoint @@ -450,14 +401,7 @@ tests: object: *utilCollection name: deleteMany arguments: - filter: {} - - - - object: *utilCollection - name: deleteMany - arguments: - documents: - - { _id: 1, x: 11 } + filter: {} - name: failPoint @@ -506,14 +450,7 @@ tests: object: *utilCollection name: deleteMany arguments: - filter: {} - - - - object: *utilCollection - name: deleteMany - arguments: - documents: - - { _id: 1, x: 11 } + filter: {} - name: failPoint @@ -562,14 +499,7 @@ tests: object: *utilCollection name: deleteMany arguments: - filter: {} - - - - object: *utilCollection - name: deleteMany - arguments: - documents: - - { _id: 1, x: 11 } + filter: {} - name: failPoint @@ -617,14 +547,7 @@ tests: object: *utilCollection name: deleteMany arguments: - filter: {} - - - - object: *utilCollection - name: deleteMany - arguments: - documents: - - { _id: 1, x: 11 } + filter: {} - name: failPoint @@ -672,14 +595,7 @@ tests: object: *utilCollection name: deleteMany arguments: - filter: {} - - - - object: *utilCollection - name: deleteMany - arguments: - documents: - - { _id: 1, x: 11 } + filter: {} - name: failPoint @@ -725,14 +641,7 @@ tests: object: *utilCollection name: deleteMany arguments: - filter: {} - - - - object: *utilCollection - name: deleteMany - arguments: - documents: - - { _id: 1, x: 11 } + filter: {} - name: failPoint @@ -781,14 +690,7 @@ tests: object: *utilCollection name: deleteMany arguments: - filter: {} - - - - object: *utilCollection - name: deleteMany - arguments: - documents: - - { _id: 1, x: 11 } + filter: {} - name: failPoint @@ -836,14 +738,7 @@ tests: object: *utilCollection name: deleteMany arguments: - filter: {} - - - - object: *utilCollection - name: deleteMany - arguments: - documents: - - { _id: 1, x: 11 } + filter: {} - name: failPoint @@ -891,14 +786,7 @@ tests: object: *utilCollection name: deleteMany arguments: - filter: {} - - - - object: *utilCollection - name: deleteMany - arguments: - documents: - - { _id: 1, x: 11 } + filter: {} - name: failPoint @@ -944,14 +832,7 @@ tests: object: *utilCollection name: deleteMany arguments: - filter: {} - - - - object: *utilCollection - name: deleteMany - arguments: - documents: - - { _id: 1, x: 11 } + filter: {} - name: failPoint @@ -997,14 +878,7 @@ tests: object: *utilCollection name: deleteMany arguments: - filter: {} - - - - object: *utilCollection - name: deleteMany - arguments: - documents: - - { _id: 1, x: 11 } + filter: {} - name: failPoint @@ -1053,14 +927,7 @@ tests: object: *utilCollection name: deleteMany arguments: - filter: {} - - - - object: *utilCollection - name: deleteMany - arguments: - documents: - - { _id: 1, x: 11 } + filter: {} - name: failPoint @@ -1108,14 +975,7 @@ tests: object: *utilCollection name: deleteMany arguments: - filter: {} - - - - object: *utilCollection - name: deleteMany - arguments: - documents: - - { _id: 1, x: 11 } + filter: {} - name: failPoint @@ -1164,14 +1024,7 @@ tests: object: *utilCollection name: deleteMany arguments: - filter: {} - - - - object: *utilCollection - name: deleteMany - arguments: - documents: - - { _id: 1, x: 11 } + filter: {} - name: failPoint @@ -1219,14 +1072,7 @@ tests: object: *utilCollection name: deleteMany arguments: - filter: {} - - - - object: *utilCollection - name: deleteMany - arguments: - documents: - - { _id: 1, x: 11 } + filter: {} - name: failPoint @@ -1274,14 +1120,7 @@ tests: object: *utilCollection name: deleteMany arguments: - filter: {} - - - - object: *utilCollection - name: deleteMany - arguments: - documents: - - { _id: 1, x: 11 } + filter: {} - name: failPoint @@ -1330,14 +1169,7 @@ tests: object: *utilCollection name: deleteMany arguments: - filter: {} - - - - object: *utilCollection - name: deleteMany - arguments: - documents: - - { _id: 1, x: 11 } + filter: {} - name: failPoint @@ -1386,14 +1218,7 @@ tests: object: *utilCollection name: deleteMany arguments: - filter: {} - - - - object: *utilCollection - name: deleteMany - arguments: - documents: - - { _id: 1, x: 11 } + filter: {} - name: failPoint @@ -1442,14 +1267,7 @@ tests: object: *utilCollection name: deleteMany arguments: - filter: {} - - - - object: *utilCollection - name: deleteMany - arguments: - documents: - - { _id: 1, x: 11 } + filter: {} - name: failPoint @@ -1497,14 +1315,7 @@ tests: object: *utilCollection name: deleteMany arguments: - filter: {} - - - - object: *utilCollection - name: deleteMany - arguments: - documents: - - { _id: 1, x: 11 } + filter: {} - name: failPoint @@ -1553,14 +1364,7 @@ tests: object: *utilCollection name: deleteMany arguments: - filter: {} - - - - object: *utilCollection - name: deleteMany - arguments: - documents: - - { _id: 1, x: 11 } + filter: {} - name: failPoint @@ -1609,14 +1413,7 @@ tests: object: *utilCollection name: deleteMany arguments: - filter: {} - - - - object: *utilCollection - name: deleteMany - arguments: - documents: - - { _id: 1, x: 11 } + filter: {} - name: failPoint @@ -1666,14 +1463,7 @@ tests: object: *utilCollection name: deleteMany arguments: - filter: {} - - - - object: *utilCollection - name: deleteMany - arguments: - documents: - - { _id: 1, x: 11 } + filter: {} - name: failPoint @@ -1723,13 +1513,6 @@ tests: name: deleteMany arguments: filter: {} - - - - object: *utilCollection - name: deleteMany - arguments: - documents: - - { _id: 1, x: 11 } - object: *utilCollection name: createIndex @@ -1783,14 +1566,7 @@ tests: object: *utilCollection name: deleteMany arguments: - filter: {} - - - - object: *utilCollection - name: deleteMany - arguments: - documents: - - { _id: 1, x: 11 } + filter: {} - name: failPoint diff --git a/source/client-backpressure/tests/backpressure-retry-loop.yml.template b/source/client-backpressure/tests/backpressure-retry-loop.yml.template index db3ce498e9..53b8ba3cf6 100644 --- a/source/client-backpressure/tests/backpressure-retry-loop.yml.template +++ b/source/client-backpressure/tests/backpressure-retry-loop.yml.template @@ -67,13 +67,6 @@ tests: arguments: filter: {} - - - object: *utilCollection - name: deleteMany - arguments: - documents: - - { _id: 1, x: 11 } - {%- if operation.operation_name == "dropIndex" %} - object: *utilCollection