From 588f1f2ff406239b178ff035726e524025ec72c3 Mon Sep 17 00:00:00 2001 From: bailey Date: Tue, 25 Nov 2025 16:00:54 -0700 Subject: [PATCH 01/19] initial commit --- source/index.md | 1 + source/mongodb-handshake/handshake.md | 1 + source/retryable-reads/retryable-reads.md | 13 +++++++++++++ source/retryable-writes/retryable-writes.md | 19 +++++++++++++++---- 4 files changed, 30 insertions(+), 4 deletions(-) diff --git a/source/index.md b/source/index.md index 29b2a51e86..bcff8b0ffd 100644 --- a/source/index.md +++ b/source/index.md @@ -12,6 +12,7 @@ - [CRUD API](crud/crud.md) - [Causal Consistency Specification](causal-consistency/causal-consistency.md) - [Change Streams](change-streams/change-streams.md) +- [Client Backpressure](client-backpressure/client-backpressure.md) - [Client Side Encryption](client-side-encryption/client-side-encryption.md) - [Client Side Operations Timeout](client-side-operations-timeout/client-side-operations-timeout.md) - [Collation](collation/collation.md) diff --git a/source/mongodb-handshake/handshake.md b/source/mongodb-handshake/handshake.md index 24c6eea50d..5a62627326 100644 --- a/source/mongodb-handshake/handshake.md +++ b/source/mongodb-handshake/handshake.md @@ -84,6 +84,7 @@ if stable_api_configured or client_options.load_balanced: else: cmd = {"legacy hello": 1, "helloOk": 1} conn.supports_op_msg = False # Send the initial command via OP_QUERY. +cmd["backpressure"] = True cmd["client"] = client_metadata if client_options.compressors: cmd["compression"] = client_options.compressors diff --git a/source/retryable-reads/retryable-reads.md b/source/retryable-reads/retryable-reads.md index d715f774ef..a7100cbe92 100644 --- a/source/retryable-reads/retryable-reads.md +++ b/source/retryable-reads/retryable-reads.md @@ -15,6 +15,10 @@ This specification will - outline how an API for retryable read operations will be implemented in drivers - define an option to enable retryable reads for an application. +The changes in this specification are related to but distinct from the retryability behaviors defined in the client +backpressure specification, which defines a retryability mechanism for all commands under certain server conditions. +Unless otherwise noted, the changes in this specification refer only to the retryability behaviors summarized above. + ## META The keywords "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and @@ -267,6 +271,13 @@ The following pseudocode for executing retryable read commands has been adapted [the pseudocode for executing retryable write commands](../retryable-writes/retryable-writes.md#executing-retryable-write-commands) and reflects the flow described above. +> [!NOTE] +> The rules above and the pseudocode below only demonstrate the rules for retryable reads as outlined in this +> specification. For simplicity, and to make the retryable reads rules easier to follow, the pseudocode was +> intentionally unmodified. For a pseudocode block that contains both retryable reads logic as defined in this +> specification and backoff retryabilitity as defined in the client backpressure specification, see the pseudocode in +> the [Backpressure Specification](../client-backpressure/client-backpressure.md). + ```typescript /** * Checks if a connection supports retryable reads. @@ -547,6 +558,8 @@ any customers experiencing degraded performance can simply disable `retryableRea ## Changelog +- xxxx-xx-xx: Clarify handling of deprioritized servers in pseudocode. + - 2024-04-30: Migrated from reStructuredText to Markdown. - 2023-12-05: Add that any server information associated with retryable exceptions MUST reflect the originating server, diff --git a/source/retryable-writes/retryable-writes.md b/source/retryable-writes/retryable-writes.md index 609de18b92..fa613a2908 100644 --- a/source/retryable-writes/retryable-writes.md +++ b/source/retryable-writes/retryable-writes.md @@ -19,6 +19,10 @@ specification will outline how an API for retryable write operations will be imp will define an option to enable retryable writes for an application and describe how a transaction ID will be provided to write commands executed therein. +The changes in this specification are related to but distinct from the retryability behaviors defined in the client +backpressure specification, which defines a retryability mechanism for all commands under certain server conditions. +Unless otherwise noted, the changes in this specification refer only to the retryability behaviors summarized above. + ## META The keywords "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and @@ -43,10 +47,10 @@ specification. This object is always associated with a server session; however, that creating a ClientSession will not always entail creation of a new server session. The name of this object MAY vary across drivers. -**Retryable Error** +**Retryable Write Error** An error is considered retryable if it has a RetryableWriteError label in its top-level "errorLabels" field. See -[Determining Retryable Errors](#determining-retryable-errors) for more information. +[Determining Retryable Write Errors](#determining-retryable-errors) for more information. Additional terms may be defined in the [Driver Session](../sessions/driver-sessions.md) specification. @@ -102,7 +106,7 @@ In a sharded cluster, it is possible that mongos may appear to support retryable cluster do not (e.g. replica set shard is configured with feature compatibility version 3.4, a standalone is added as a new shard). In these rare cases, a write command that fans out to a shard that does not support retryable writes may partially fail and an error may be reported in the write result from mongos (e.g. `writeErrors` array in the bulk write -result). This does not constitute a retryable error. Drivers MUST relay such errors to the user. +result). This does not constitute a retryable write error. Drivers MUST relay such errors to the user. #### Supported Write Operations @@ -162,7 +166,7 @@ occurs during a write command within a transaction (excepting `commitTransation` ### Implementing Retryable Writes -#### Determining Retryable Errors +#### Determining Retryable Write Errors When connected to a MongoDB instance that supports retryable writes (versions 3.6+), the driver MUST treat all errors with the RetryableWriteError label as retryable. This error label can be found in the top-level "errorLabels" field of @@ -333,6 +337,13 @@ errors are labeled "NoWritesPerformed", then the first error should be raised. If a driver associates server information (e.g. the server address or description) with an error, the driver MUST ensure that the reported server information corresponds to the server that originated the error. +> [!NOTE] +> The rules above and the pseudocode below only demonstrate the rules for retryable writes as outlined in this +> specification. For simplicity, and to make the retryable writes rules easier to follow, the pseudocode was +> intentionally unmodified. For a pseudocode block that contains both retryable writes logic as defined in this +> specification and backoff retryabilitity as defined in the client backpressure specification, see the pseudocode in +> the [Backpressure Specification](../client-backpressure/client-backpressure.md). + The above rules are implemented in the following pseudo-code: ```typescript From e467f5bc0409b550ba3f392e6b163f97337ba901 Mon Sep 17 00:00:00 2001 From: bailey Date: Wed, 26 Nov 2025 10:15:49 -0700 Subject: [PATCH 02/19] new files --- .../client-backpressure.md | 424 ++++++++++++++++++ source/client-backpressure/tests/README.md | 15 + 2 files changed, 439 insertions(+) create mode 100644 source/client-backpressure/client-backpressure.md create mode 100644 source/client-backpressure/tests/README.md diff --git a/source/client-backpressure/client-backpressure.md b/source/client-backpressure/client-backpressure.md new file mode 100644 index 0000000000..88dd94807e --- /dev/null +++ b/source/client-backpressure/client-backpressure.md @@ -0,0 +1,424 @@ +# Client Backpressure + +- Status: Accepted +- Minimum Server Version: N/A + +______________________________________________________________________ + +## Abstract + +This specification adds the ability for drivers to automatically retry requests that fail due to server overload errors +while applying backpressure to avoid further overloading the server. + +## META + +The keywords "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and +"OPTIONAL" in this document are to be interpreted as described in [RFC 2119](https://www.ietf.org/rfc/rfc2119.txt). + +## Specification + +### Terms + +#### Ingress Connection Rate Limiter + +A token-bucket based system introduced in MongoDB 8.2 to admit, reject or queue connection requests. It aims to prevent +connection spikes from overloading the system. + +#### Ingress Request Rate Limiter + +A token bucket based system introduced in MongoDB 8.2 to admit an operation or reject it with a System Overload Error at +the front door of a mongod/s. It aims to prevent operations spikes from overloading the system. + +#### MongoTune + +Mongotune is a policy engine outside the server (mongod or mongos) which monitors a set of metrics (MongoDB or system +host) to dynamically configure MongoDB settings. MongoTune is deployed to Atlas clusters and will dynamically configure +the connection and request rate limiters to prevent and mitigate overloading the system. + +#### RetryableError label + +An error is considered retryable if it includes the "RetryableError" label. This error label indicates that an operation +is safely retryable regardless of the type of operation, its metadata, or any of its arguments. + +#### SystemOverloadedError label + +An error is considered overloaded if it includes the "SystemOverloadError" label. This error label indicates that the +server is overloaded. If this error label is present, drivers will backoff before attempting a retry. + +#### Overload Errors + +An overload error is any command or network error that occurs due to a server overload. For example, when a request +exceeds the ingress request rate limit: + +```js +{ + 'ok': 0.0, + 'errmsg': "Rate limiter 'ingressRequestRateLimiter' rate exceeded", + 'code': 462, + 'codeName': 'IngressRequestRateLimitExceeded', + 'errorLabels': ['SystemOverloadedError', 'RetryableError'], +} +``` + +When a new connection attempt exceeds the ingress connection rate limit, the server closes the TCP connection before TLS +handshake is complete. Drivers will observe this as a network error (e.g. "connection reset by peer" or "connection +closed"). + +When a new connection attempt is queued by the server for so long that the driver-side timeout expires, drivers will +observe this as a network timeout error. + +### Requirements for Client Backpressure + +#### Overload retry policy + +This specification expands the driver's retry ability to all commands, including those not currently considered +retryable such as updateMany, create collection, getMore, and generic runCommand. The new command execution method obeys +the following rules: + +1. If the command succeeds on the first attempt, drivers MUST deposit `RETRY_TOKEN_RETURN_RATE` tokens. + - The value is 0.1 and non-configurable. +2. If the command succeeds on a retry attempt, drivers MUST deposit `RETRY_TOKEN_RETURN_RATE`+1 tokens. +3. If a retry attempt fails with an error that does not include `SystemOverloadedError` label, drivers MUST deposit 1 + token. +4. A retry attempt will only be permitted if the error includes the `RetryableError` label, we have not reached + `MAX_ATTEMPTS`, the CSOT deadline has not expired, and a token can be acquired from the token bucket. + - The value of `MAX_ATTEMPTS` is 5 and non-configurable. + - This intentionally changes the behavior of CSOT which otherwise would retry an unlimited number of times within the + timeout to avoid retry storms. +5. If the previous error includes the `SystemOverloadedError` label, the client MUST apply exponential backoff according + to according to the following formula: `delayMS = j * min(maxBackoff, baseBackoff * 2^i)` + - `i` is the retry attempt (starting with 0 for the first retry). + - `j` is a random jitter value between 0 and 1. + - `baseBackoff` is constant 100ms. + - `maxBackoff` is 10000ms. + - This results in delays of 100ms, 200ms, 400ms, 800ms, and 1600ms before accounting for jitter. +6. If the previous error contained the `SystemOverloadedError` error label, the node will be added to the set of + deprioritized servers. + +#### Pseudocode + +The following pseudocode describes the overload retry policy: + +```python +BASE_BACKOFF = 0.1 +MAX_BACKOFF = 10 +RETRY_TOKEN_RETURN_RATE = 0.1 + +def execute_command_retryable(command, ...): + deprioritized_servers = [] + attempt = 0 + while True: + try: + server = select_server(deprioritized_servers) + connection = server.getConnection() + res = execute_command(connection, command) + # Return tokens to the bucket on success. + tokens = RETRY_TOKEN_RETURN_RATE + if attempt > 0: + tokens += 1 + token_bucket.deposit(tokens) + return res + except PyMongoError as exc: + backoff = 0 + attempt += 1 + + if attempt > MAX_ATTEMPTS: + raise + + # Raise if the error is non retryable. + is_retryable = exc.has_error_label("RetryableError") or is_retryable_write_error() or is_retryable_read_error() + if not is_retryable: + raise error + if exc.has_error_label("SystemOverloadedError"): + jitter = random.random() # Random float between [0.0, 1.0). + backoff = jitter * min(BASE_BACKOFF * (2 ** attempt), MAX_BACKOFF) + + # If the delay exceeds the deadline, bail early before consuming a token. + if _csot.get_timeout(): + if time.monotonic() + backoff > _csot.get_deadline(): + raise + + if not token_bucket.consume(1): + raise + + if backoff: + time.sleep(backoff) + deprioritized_servers.append(server) + continue +``` + +Some drivers might not have retryability implementations that allow easy separation of the existing retryable +reads/writes mechanisms from the exponential backoff and jitter retry algorithm. An example pseudocode is defined below +that demonstrates a combined retryable reads/writes implementation with the corresponding backpressure changes (adapted +from the Node driver's implementation): + +```typescript +async function tryOperation>( + operation: T, + { topology, timeoutContext, session, readPreference }: RetryOptions +): Promise { + const serverSelector = getServerSelectorForReadPreference(operation, readPreference); + + let server = await topology.selectServer(selector, { + session, + }); + + const hasReadAspect = operation.hasAspect(Aspect.READ_OPERATION); + const hasWriteAspect = operation.hasAspect(Aspect.WRITE_OPERATION); + const inTransaction = session?.inTransaction() ?? false; + + const willRetryRead = topology.s.options.retryReads && !inTransaction && operation.canRetryRead; + + const willRetryWrite = + topology.s.options.retryWrites && + !inTransaction && + supportsRetryableWrites(server) && + operation.canRetryWrite; + + const willRetry = + operation.hasAspect(Aspect.RETRYABLE) && + session != null && + ((hasReadAspect && willRetryRead) || (hasWriteAspect && willRetryWrite)); + + if (hasWriteAspect && willRetryWrite && session != null) { + operation.options.willRetryWrite = true; + session.incrementTransactionNumber(); + } + + // The maximum number of retry attempts using regular retryable reads/writes logic (not including + // SystemOverLoad error retries). + const maxNonOverloadRetryAttempts = willRetry + ? timeoutMS != null + ? Infinity + : 2 + : 1; + + let previousOperationError: MongoError | undefined; + let previousServer: ServerDescription | undefined; + + let nonOverloadRetryAttempt = 0; + let systemOverloadRetryAttempt = 0; + + const maxSystemOverloadRetryAttempts = 5; + const backoffDelayProvider = exponentialBackoffDelayProvider( + 10_000, // MAX_BACKOFF + 100, // base backoff + 2 // backoff rate + ); + + while (true) { + if (previousOperationError) { + if (previousOperationError.hasErrorLabel("SystemOverloadError")) { + systemOverloadRetryAttempt += 1; + + if ( + // if the SystemOverloadError is not retryable, throw. + !previousOperationError.hasErrorLabel("RetryableError") || + !( + // if retryable writes or reads are not configured, throw. + ( + (hasReadAspect && topology.s.options.retryReads) || + (hasWriteAspect && topology.s.options.retryWrites) + ) + ) + ) { + throw previousOperationError; + } + + // if we have exhausted overload retry attempts, throw. + if (systemOverloadRetryAttempt > maxSystemOverloadRetryAttempts) { + throw previousOperationError; + } + + const { value: delayMS } = backoffDelayProvider.next(); + + // if the delay would exhaust the CSOT timeout, short-circuit. + if (timeoutContext.csotEnabled() && delayMS > timeoutContext.remainingTimeMS) { + throw previousError; + } + + await setTimeout(delayMS); + + // attempt to consume a retry token, throw if we don't have budget. + if (!topology.tokenBucket.consume(RETRY_COST)) { + throw previousOperationError; + } + + server = await topology.selectServer(selector, { session }); + } else { + nonOverloadRetryAttempt++; + // we have no more retry attempts, throw. + if (nonOverloadRetryAttempt > maxNonOverloadRetryAttempts) { + throw previousOperationError; + } + + // Handle MMAPv1 not supporting retryable writes. + if (hasWriteAspect && previousOperationError.code === MMAPv1_RETRY_WRITES_ERROR_CODE) { + throw new MongoServerError({ + message: MMAPv1_RETRY_WRITES_ERROR_MESSAGE, + errmsg: MMAPv1_RETRY_WRITES_ERROR_MESSAGE, + originalError: previousOperationError + }); + } + + // handle non-retryable errors + if ( + (hasWriteAspect && !isRetryableWriteError(previousOperationError)) || + (hasReadAspect && !isRetryableReadError(previousOperationError)) + ) { + throw previousOperationError; + } + + server = await topology.selectServer(selector, { session }); + + // handle rare downgrade scenarios where some nodes don't support + // retryable writes but others do. + if (hasWriteAspect && !supportsRetryableWrites(server)) { + throw new MongoUnexpectedServerResponseError( + 'Selected server does not support retryable writes' + ); + } + } + } + + try { + try { + const result = await server.command(operation, timeoutContext); + const isRetry = nonOverloadRetryAttempt > 0 || systemOverloadRetryAttempt > 0; + topology.tokenBucket.deposit( + isRetry + ? // on successful retry, deposit the retry cost + the refresh rate. + TOKEN_REFRESH_RATE + RETRY_COST + : // otherwise, just deposit the refresh rate. + TOKEN_REFRESH_RATE + ); + return operation.handleOk(result); + } catch (error) { + return operation.handleError(error); + } + } catch (operationError) { + if (!operationError.hasErrorLabel("SystemOverloadError")) { + // if an operation fails with an error that does not contain the SystemOverloadError, deposit 1 token. + topology.tokenBucket.deposit(RETRY_COST); + } + + if ( + previousOperationError != null && + operationError.hasErrorLabel("NoWritesPerformed") + ) { + throw previousOperationError; + } + previousServer = server.description; + previousOperationError = operationError; + } + } +} +``` + +### Token Bucket + +The overload retry policy introduces a per-client token bucket to limit retry attempts. Although the server rejects +excess operations as quickly as possible, doing so costs CPU and creates extra contention on the connection pool which +can eventually negatively affect goodput. To reduce this risk, the token bucket will limit retry attempts during a +prolonged overload. + +The token bucket capacity is set to 1000 for consistency with the server. + +#### Pseudocode + +The token bucket is implemented via a thread safe counter. For languages without atomics, this can be implemented via a +lock, for example: + +```python +DEFAULT_RETRY_TOKEN_CAPACITY = 1000 +class TokenBucket: + """A token bucket implementation for rate limiting.""" + def __init__( + self, + capacity: float = DEFAULT_RETRY_TOKEN_CAPACITY, + ): + self.lock = Lock() + self.capacity = capacity + self.tokens = capacity + + def consume(self, n: float) -> bool: + """Consume n tokens from the bucket if available.""" + with self.lock: + if self.tokens >= n: + self.tokens -= n + return True + return False + + def deposit(self, n: float) -> None: + """Deposit n tokens back into the bucket.""" + with self.lock: + self.tokens = min(self.capacity, self.tokens + n) +``` + +#### Handshake changes + +Drivers conforming to this spec MUST add `“backpressure”: True` to the connection handshake. This flag allows the server +to identify clients which do and do not support backpressure. Currently, this flag is unused but in the future the +server may offer different rate limiting behavior for clients that do not support backpressure. + +##### Implementation notes + +On some platforms sleep() can have a very low precision, meaning an attempt to sleep for 50ms may actually sleep for a +much larger time frame. Drivers are not required to work around this limitation. + +### Logging Retry Attempts + +[As with retryable writes](../retryable-writes/retryable-writes.md#logging-retry-attempts), drivers MAY choose to log +retry attempts for load shed operations. This specification does not define a format for such log messages. + +### Command Monitoring + +[As with retryable writes](../retryable-writes/retryable-writes.md#command-monitoring), in accordance with the +[Command Logging and Monitoring](../command-logging-and-monitoring/command-logging-and-monitoring.md) specification, +drivers MUST guarantee that each `CommandStartedEvent` has either a correlating `CommandSucceededEvent` or +`CommandFailedEvent` and that every "command started" log message has either a correlating "command succeeded" log +message or "command failed" log message. If the first attempt of a retryable operation encounters a retryable error, +drivers MUST fire a `CommandFailedEvent` and emit a "command failed" log message for the retryable error and fire a +separate `CommandStartedEvent` and emit a separate "command started" log message when executing the subsequent retry +attempt. Note that the second `CommandStartedEvent` and "command started" log message may have a different +`connectionId`, since a server is reselected for a retry attempt. + +### Documentation + +1. Drivers MUST document that all operations support retries on server overload. +2. Driver release notes MUST make it clear to users that they may need to adjust custom retry logic to prevent an + application from inadvertently retrying for too long (see [Backwards Compatibility](#backwards-compatibility) for + details). + +## Test Plan + +See the [README](./tests/README.md) for tests. + +## Motivation for Change + +New load shedding mechanisms are being introduced to the server that improve its ability to remain available under +extreme load, however clients do not know how to handle the errors returned when one of its requests has been rejected. +As a result, such overload errors would currently either be propagated back to applications, increasing +externally-visible command failure rates, or be retried immediately, increasing the load on already overburdened +servers. To minimize these effects, this specification enables clients to retry requests that have been load shed in a +way that does not overburden already overloaded servers. This retry behavior allows for more aggressive and effective +load shedding policies to be deployed in the future. This will also help unify the currently-divergent retry behavior +between drivers and the server (mongos). + +## Reference Implementation + +The Node and Python drivers will provide the reference implementations. See +[NODE-7142](https://jira.mongodb.org/browse/NODE-7142) and [PYTHON-5528](https://jira.mongodb.org/browse/PYTHON-5528). + +## Future work + +1. [DRIVERS-3333](https://jira.mongodb.org/browse/DRIVERS-3333) Add a backoff state into the connection pool. +2. [DRIVERS-3241](https://jira.mongodb.org/browse/DRIVERS-3241) Add diagnostic metadata to retried commands. + +## Q&A + +TODO + +## Changelog + +- 2025-XX-XX: Initial version. diff --git a/source/client-backpressure/tests/README.md b/source/client-backpressure/tests/README.md new file mode 100644 index 0000000000..7a70e4ad76 --- /dev/null +++ b/source/client-backpressure/tests/README.md @@ -0,0 +1,15 @@ +# Client Backpressure Tests + +______________________________________________________________________ + +## Introduction + +The YAML and JSON files in this directory are platform-independent tests meant to exercise a driver's implementation of +retryable reads. These tests utilize the [Unified Test Format](../../unified-test-format/unified-test-format.md). + +Several prose tests, which are not easily expressed in YAML, are also presented in this file. Those tests will need to +be manually implemented by each driver. + +## Changelog + +- 2025-XX-XX: Initial version. From d55fdb9ebbcecc5f9f15d16b8d9d5151495b3713 Mon Sep 17 00:00:00 2001 From: bailey Date: Wed, 26 Nov 2025 14:25:13 -0700 Subject: [PATCH 03/19] add tests for handshake changes --- source/mongodb-handshake/tests/README.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/source/mongodb-handshake/tests/README.md b/source/mongodb-handshake/tests/README.md index d88a49fa92..0ba58b713a 100644 --- a/source/mongodb-handshake/tests/README.md +++ b/source/mongodb-handshake/tests/README.md @@ -486,3 +486,17 @@ Before each test case, perform the setup. 7. Store the response as `updatedClientMetadata`. 8. Assert that `initialClientMetadata` is identical to `updatedClientMetadata`. + +### Test 9: Handshake documents include `backpressure: true` + +These tests require a mechanism for observing handshake documents sent to the server. + +1. Create a `MongoClient` that is configured to record all handshake documents sent to the server as a part of + connection establishment. + +2. Send a `ping` command to the server and verify that the command succeeds. This ensure that a connection is + established on all topologies. + +3. Assert that for every handshake document intercepted: + + 1. the document has a field `backpressure` whose value is `true`. From 8e74b418404a573d6d622122e031b088871d169b Mon Sep 17 00:00:00 2001 From: bailey Date: Mon, 1 Dec 2025 15:59:14 -0700 Subject: [PATCH 04/19] add generated tests --- .../tests/backpressure-retry-loop.json | 3403 +++++++++++++++++ .../tests/backpressure-retry-loop.yml | 1860 +++++++++ .../backpressure-retry-loop.yml.template | 128 + .../backpressure-retry-max-attempts.json | 1262 ++++++ .../tests/backpressure-retry-max-attempts.yml | 753 ++++ ...ckpressure-retry-max-attempts.yml.template | 72 + ...enerate-backpressure-retryability-tests.py | 125 + 7 files changed, 7603 insertions(+) create mode 100644 source/client-backpressure/tests/backpressure-retry-loop.json create mode 100644 source/client-backpressure/tests/backpressure-retry-loop.yml create mode 100644 source/client-backpressure/tests/backpressure-retry-loop.yml.template create mode 100644 source/client-backpressure/tests/backpressure-retry-max-attempts.json create mode 100644 source/client-backpressure/tests/backpressure-retry-max-attempts.yml create mode 100644 source/client-backpressure/tests/backpressure-retry-max-attempts.yml.template create mode 100644 source/etc/generate-backpressure-retryability-tests.py diff --git a/source/client-backpressure/tests/backpressure-retry-loop.json b/source/client-backpressure/tests/backpressure-retry-loop.json new file mode 100644 index 0000000000..21fc802344 --- /dev/null +++ b/source/client-backpressure/tests/backpressure-retry-loop.json @@ -0,0 +1,3403 @@ +{ + "description": "tests that operations respect overload backoff retry loop", + "schemaVersion": "1.0", + "runOnRequirements": [ + { + "minServerVersion": "4.4", + "topologies": [ + "replicaset", + "sharded", + "load-balanced" + ] + } + ], + "createEntities": [ + { + "client": { + "id": "client", + "useMultipleMongoses": false, + "observeEvents": [ + "commandStartedEvent", + "commandSucceededEvent", + "commandFailedEvent" + ] + } + }, + { + "client": { + "id": "failPointClient", + "useMultipleMongoses": false + } + }, + { + "database": { + "id": "utilDb", + "client": "failPointClient", + "databaseName": "retryable-writes-tests" + } + }, + { + "collection": { + "id": "utilCollection", + "database": "utilDb", + "collectionName": "coll" + } + }, + { + "database": { + "id": "database", + "client": "client", + "databaseName": "retryable-writes-tests" + } + }, + { + "collection": { + "id": "collection", + "database": "database", + "collectionName": "coll" + } + } + ], + "initialData": [ + { + "collectionName": "coll", + "databaseName": "retryable-writes-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + } + ] + } + ], + "tests": [ + { + "description": "client.listDatabases retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "listDatabases" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "client", + "name": "listDatabases", + "arguments": { + "filter": {} + }, + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandSucceededEvent": { + "commandName": "listDatabases" + } + } + ] + } + ] + }, + { + "description": "client.listDatabaseNames retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "listDatabases" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "client", + "name": "listDatabaseNames", + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandSucceededEvent": { + "commandName": "listDatabases" + } + } + ] + } + ] + }, + { + "description": "client.createChangeStream retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "client", + "name": "createChangeStream", + "arguments": { + "pipeline": [] + }, + "saveResultAsEntity": "changeStream", + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandSucceededEvent": { + "commandName": "aggregate" + } + } + ] + } + ] + }, + { + "description": "client.clientBulkWrite retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "bulkWrite" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "client", + "name": "clientBulkWrite", + "arguments": { + "models": [ + { + "insertOne": { + "namespace": "retryable-writes-tests.coll", + "document": { + "_id": 8, + "x": 88 + } + } + } + ] + }, + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandFailedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandStartedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandFailedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandStartedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandFailedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandStartedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandSucceededEvent": { + "commandName": "bulkWrite" + } + } + ] + } + ] + }, + { + "description": "database.aggregate retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "database", + "name": "aggregate", + "arguments": { + "pipeline": [ + { + "$listLocalSessions": {} + }, + { + "$limit": 1 + } + ] + }, + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandSucceededEvent": { + "commandName": "aggregate" + } + } + ] + } + ] + }, + { + "description": "database.listCollections retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "listCollections" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "database", + "name": "listCollections", + "arguments": { + "filter": {} + }, + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandSucceededEvent": { + "commandName": "listCollections" + } + } + ] + } + ] + }, + { + "description": "database.listCollectionNames retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "listCollections" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "database", + "name": "listCollectionNames", + "arguments": { + "filter": {} + }, + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandSucceededEvent": { + "commandName": "listCollections" + } + } + ] + } + ] + }, + { + "description": "database.runCommand retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "ping" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "database", + "name": "runCommand", + "arguments": { + "command": { + "ping": 1 + }, + "commandName": "ping" + }, + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "ping" + } + }, + { + "commandFailedEvent": { + "commandName": "ping" + } + }, + { + "commandStartedEvent": { + "commandName": "ping" + } + }, + { + "commandFailedEvent": { + "commandName": "ping" + } + }, + { + "commandStartedEvent": { + "commandName": "ping" + } + }, + { + "commandFailedEvent": { + "commandName": "ping" + } + }, + { + "commandStartedEvent": { + "commandName": "ping" + } + }, + { + "commandSucceededEvent": { + "commandName": "ping" + } + } + ] + } + ] + }, + { + "description": "database.createChangeStream retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "database", + "name": "createChangeStream", + "arguments": { + "pipeline": [] + }, + "saveResultAsEntity": "changeStream", + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandSucceededEvent": { + "commandName": "aggregate" + } + } + ] + } + ] + }, + { + "description": "collection.aggregate retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "aggregate", + "arguments": { + "pipeline": [] + }, + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandSucceededEvent": { + "commandName": "aggregate" + } + } + ] + } + ] + }, + { + "description": "collection.countDocuments retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "countDocuments", + "arguments": { + "filter": {} + }, + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandSucceededEvent": { + "commandName": "aggregate" + } + } + ] + } + ] + }, + { + "description": "collection.estimatedDocumentCount retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "count" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "estimatedDocumentCount", + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "count" + } + }, + { + "commandFailedEvent": { + "commandName": "count" + } + }, + { + "commandStartedEvent": { + "commandName": "count" + } + }, + { + "commandFailedEvent": { + "commandName": "count" + } + }, + { + "commandStartedEvent": { + "commandName": "count" + } + }, + { + "commandFailedEvent": { + "commandName": "count" + } + }, + { + "commandStartedEvent": { + "commandName": "count" + } + }, + { + "commandSucceededEvent": { + "commandName": "count" + } + } + ] + } + ] + }, + { + "description": "collection.distinct retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "distinct" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "distinct", + "arguments": { + "fieldName": "x", + "filter": {} + }, + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "distinct" + } + }, + { + "commandFailedEvent": { + "commandName": "distinct" + } + }, + { + "commandStartedEvent": { + "commandName": "distinct" + } + }, + { + "commandFailedEvent": { + "commandName": "distinct" + } + }, + { + "commandStartedEvent": { + "commandName": "distinct" + } + }, + { + "commandFailedEvent": { + "commandName": "distinct" + } + }, + { + "commandStartedEvent": { + "commandName": "distinct" + } + }, + { + "commandSucceededEvent": { + "commandName": "distinct" + } + } + ] + } + ] + }, + { + "description": "collection.find retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "find" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "find", + "arguments": { + "filter": {} + }, + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandSucceededEvent": { + "commandName": "find" + } + } + ] + } + ] + }, + { + "description": "collection.findOne retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "find" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "findOne", + "arguments": { + "filter": {} + }, + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandSucceededEvent": { + "commandName": "find" + } + } + ] + } + ] + }, + { + "description": "collection.listIndexes retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "listIndexes" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "listIndexes", + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandSucceededEvent": { + "commandName": "listIndexes" + } + } + ] + } + ] + }, + { + "description": "collection.listIndexNames retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "listIndexes" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "listIndexNames", + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandSucceededEvent": { + "commandName": "listIndexes" + } + } + ] + } + ] + }, + { + "description": "collection.createChangeStream retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "createChangeStream", + "arguments": { + "pipeline": [] + }, + "saveResultAsEntity": "changeStream", + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandSucceededEvent": { + "commandName": "aggregate" + } + } + ] + } + ] + }, + { + "description": "collection.insertOne retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "insert" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "insertOne", + "arguments": { + "document": { + "_id": 2, + "x": 22 + } + }, + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandSucceededEvent": { + "commandName": "insert" + } + } + ] + } + ] + }, + { + "description": "collection.insertMany retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "insert" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "insertMany", + "arguments": { + "documents": [ + { + "_id": 2, + "x": 22 + } + ] + }, + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandSucceededEvent": { + "commandName": "insert" + } + } + ] + } + ] + }, + { + "description": "collection.deleteOne retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "delete" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "deleteOne", + "arguments": { + "filter": {} + }, + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandSucceededEvent": { + "commandName": "delete" + } + } + ] + } + ] + }, + { + "description": "collection.deleteMany retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "delete" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "deleteMany", + "arguments": { + "filter": {} + }, + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandSucceededEvent": { + "commandName": "delete" + } + } + ] + } + ] + }, + { + "description": "collection.replaceOne retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "update" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "replaceOne", + "arguments": { + "filter": {}, + "replacement": { + "x": 22 + } + }, + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandSucceededEvent": { + "commandName": "update" + } + } + ] + } + ] + }, + { + "description": "collection.updateOne retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "update" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "updateOne", + "arguments": { + "filter": {}, + "update": { + "$set": { + "x": 22 + } + } + }, + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandSucceededEvent": { + "commandName": "update" + } + } + ] + } + ] + }, + { + "description": "collection.updateMany retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "update" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "updateMany", + "arguments": { + "filter": {}, + "update": { + "$set": { + "x": 22 + } + } + }, + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandSucceededEvent": { + "commandName": "update" + } + } + ] + } + ] + }, + { + "description": "collection.findOneAndDelete retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "findAndModify" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "findOneAndDelete", + "arguments": { + "filter": {} + }, + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandSucceededEvent": { + "commandName": "findAndModify" + } + } + ] + } + ] + }, + { + "description": "collection.findOneAndReplace retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "findAndModify" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "findOneAndReplace", + "arguments": { + "filter": {}, + "replacement": { + "x": 22 + } + }, + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandSucceededEvent": { + "commandName": "findAndModify" + } + } + ] + } + ] + }, + { + "description": "collection.findOneAndUpdate retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "findAndModify" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "findOneAndUpdate", + "arguments": { + "filter": {}, + "update": { + "$set": { + "x": 22 + } + } + }, + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandSucceededEvent": { + "commandName": "findAndModify" + } + } + ] + } + ] + }, + { + "description": "collection.bulkWrite retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "insert" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "bulkWrite", + "arguments": { + "requests": [ + { + "insertOne": { + "document": { + "_id": 2, + "x": 22 + } + } + } + ] + }, + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandSucceededEvent": { + "commandName": "insert" + } + } + ] + } + ] + }, + { + "description": "collection.createIndex retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "createIndexes" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "createIndex", + "arguments": { + "keys": { + "x": 11 + }, + "name": "x_11" + }, + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandSucceededEvent": { + "commandName": "createIndexes" + } + } + ] + } + ] + }, + { + "description": "collection.dropIndex retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "object": "utilCollection", + "name": "createIndex", + "arguments": { + "keys": { + "x": 11 + }, + "name": "x_11" + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "dropIndexes" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "dropIndex", + "arguments": { + "name": "x_11" + }, + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandSucceededEvent": { + "commandName": "dropIndexes" + } + } + ] + } + ] + }, + { + "description": "collection.dropIndexes retries using operation loop", + "operations": [ + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + }, + { + "object": "utilCollection", + "name": "deleteMany", + "arguments": { + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "dropIndexes" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "dropIndexes", + "expectError": false + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandSucceededEvent": { + "commandName": "dropIndexes" + } + } + ] + } + ] + } + ] +} diff --git a/source/client-backpressure/tests/backpressure-retry-loop.yml b/source/client-backpressure/tests/backpressure-retry-loop.yml new file mode 100644 index 0000000000..ec2fa25b8f --- /dev/null +++ b/source/client-backpressure/tests/backpressure-retry-loop.yml @@ -0,0 +1,1860 @@ +# Tests in this file are generated from backpressure-retry-loop.yml.template. + +description: tests that operations respect overload backoff retry loop + +schemaVersion: '1.0' + +runOnRequirements: + - + minServerVersion: '4.4' # failCommand + topologies: [replicaset, sharded, load-balanced] + +createEntities: + - + client: + id: &client client + useMultipleMongoses: false + observeEvents: [ 'commandStartedEvent', 'commandSucceededEvent', 'commandFailedEvent' ] + + - + client: + id: &failPointClient failPointClient + useMultipleMongoses: false + + - + database: + id: &utilDb utilDb + client: *failPointClient + databaseName: &database_name retryable-writes-tests + + - + collection: + id: &utilCollection utilCollection + database: *utilDb + collectionName: &collection_name coll + + - + database: + id: &database database + client: *client + databaseName: &database_name retryable-writes-tests + - + collection: + id: &collection collection + database: *database + collectionName: &collection_name coll + +initialData: + - + collectionName: *collection_name + databaseName: *database_name + documents: + - { _id: 1, x: 11 } + - { _id: 2, x: 22 } + +tests: + + - + description: 'client.listDatabases retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [listDatabases] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *client + name: listDatabases + arguments: + filter: {} + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandSucceededEvent: + commandName: listDatabases + + - + description: 'client.listDatabaseNames retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [listDatabases] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *client + name: listDatabaseNames + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandSucceededEvent: + commandName: listDatabases + + - + description: 'client.createChangeStream retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [aggregate] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *client + name: createChangeStream + arguments: + pipeline: [] + saveResultAsEntity: changeStream + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandSucceededEvent: + commandName: aggregate + + - + description: 'client.clientBulkWrite retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [bulkWrite] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *client + name: clientBulkWrite + arguments: + models: + - insertOne: + namespace: retryable-writes-tests.coll + document: { _id: 8, x: 88 } + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: bulkWrite + - commandFailedEvent: + commandName: bulkWrite + - commandStartedEvent: + commandName: bulkWrite + - commandFailedEvent: + commandName: bulkWrite + - commandStartedEvent: + commandName: bulkWrite + - commandFailedEvent: + commandName: bulkWrite + - commandStartedEvent: + commandName: bulkWrite + - commandSucceededEvent: + commandName: bulkWrite + + - + description: 'database.aggregate retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [aggregate] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *database + name: aggregate + arguments: + pipeline: [ { $listLocalSessions: {} }, { $limit: 1 } ] + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandSucceededEvent: + commandName: aggregate + + - + description: 'database.listCollections retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [listCollections] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *database + name: listCollections + arguments: + filter: {} + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandSucceededEvent: + commandName: listCollections + + - + description: 'database.listCollectionNames retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [listCollections] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *database + name: listCollectionNames + arguments: + filter: {} + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandSucceededEvent: + commandName: listCollections + + - + description: 'database.runCommand retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [ping] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *database + name: runCommand + arguments: + command: { ping: 1 } + commandName: ping + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: ping + - commandFailedEvent: + commandName: ping + - commandStartedEvent: + commandName: ping + - commandFailedEvent: + commandName: ping + - commandStartedEvent: + commandName: ping + - commandFailedEvent: + commandName: ping + - commandStartedEvent: + commandName: ping + - commandSucceededEvent: + commandName: ping + + - + description: 'database.createChangeStream retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [aggregate] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *database + name: createChangeStream + arguments: + pipeline: [] + saveResultAsEntity: changeStream + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandSucceededEvent: + commandName: aggregate + + - + description: 'collection.aggregate retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [aggregate] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: aggregate + arguments: + pipeline: [] + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandSucceededEvent: + commandName: aggregate + + - + description: 'collection.countDocuments retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [aggregate] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: countDocuments + arguments: + filter: {} + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandSucceededEvent: + commandName: aggregate + + - + description: 'collection.estimatedDocumentCount retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [count] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: estimatedDocumentCount + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: count + - commandFailedEvent: + commandName: count + - commandStartedEvent: + commandName: count + - commandFailedEvent: + commandName: count + - commandStartedEvent: + commandName: count + - commandFailedEvent: + commandName: count + - commandStartedEvent: + commandName: count + - commandSucceededEvent: + commandName: count + + - + description: 'collection.distinct retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [distinct] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: distinct + arguments: + fieldName: x + filter: {} + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: distinct + - commandFailedEvent: + commandName: distinct + - commandStartedEvent: + commandName: distinct + - commandFailedEvent: + commandName: distinct + - commandStartedEvent: + commandName: distinct + - commandFailedEvent: + commandName: distinct + - commandStartedEvent: + commandName: distinct + - commandSucceededEvent: + commandName: distinct + + - + description: 'collection.find retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [find] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: find + arguments: + filter: {} + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandSucceededEvent: + commandName: find + + - + description: 'collection.findOne retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [find] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: findOne + arguments: + filter: {} + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandSucceededEvent: + commandName: find + + - + description: 'collection.listIndexes retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [listIndexes] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: listIndexes + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandSucceededEvent: + commandName: listIndexes + + - + description: 'collection.listIndexNames retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [listIndexes] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: listIndexNames + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandSucceededEvent: + commandName: listIndexes + + - + description: 'collection.createChangeStream retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [aggregate] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: createChangeStream + arguments: + pipeline: [] + saveResultAsEntity: changeStream + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandSucceededEvent: + commandName: aggregate + + - + description: 'collection.insertOne retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [insert] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: insertOne + arguments: + document: { _id: 2, x: 22 } + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandSucceededEvent: + commandName: insert + + - + description: 'collection.insertMany retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [insert] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: insertMany + arguments: + documents: + - { _id: 2, x: 22 } + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandSucceededEvent: + commandName: insert + + - + description: 'collection.deleteOne retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [delete] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: deleteOne + arguments: + filter: {} + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandSucceededEvent: + commandName: delete + + - + description: 'collection.deleteMany retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [delete] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: deleteMany + arguments: + filter: {} + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandSucceededEvent: + commandName: delete + + - + description: 'collection.replaceOne retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [update] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: replaceOne + arguments: + filter: {} + replacement: { x: 22 } + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandSucceededEvent: + commandName: update + + - + description: 'collection.updateOne retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [update] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: updateOne + arguments: + filter: {} + update: { $set: { x: 22 } } + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandSucceededEvent: + commandName: update + + - + description: 'collection.updateMany retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [update] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: updateMany + arguments: + filter: {} + update: { $set: { x: 22 } } + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandSucceededEvent: + commandName: update + + - + description: 'collection.findOneAndDelete retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [findAndModify] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: findOneAndDelete + arguments: + filter: {} + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandSucceededEvent: + commandName: findAndModify + + - + description: 'collection.findOneAndReplace retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [findAndModify] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: findOneAndReplace + arguments: + filter: {} + replacement: { x: 22 } + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandSucceededEvent: + commandName: findAndModify + + - + description: 'collection.findOneAndUpdate retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [findAndModify] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: findOneAndUpdate + arguments: + filter: {} + update: { $set: { x: 22 } } + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandSucceededEvent: + commandName: findAndModify + + - + description: 'collection.bulkWrite retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [insert] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: bulkWrite + arguments: + requests: + - insertOne: + document: { _id: 2, x: 22 } + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandSucceededEvent: + commandName: insert + + - + description: 'collection.createIndex retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [createIndexes] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: createIndex + arguments: + keys: { x: 11 } + name: "x_11" + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: createIndexes + - commandFailedEvent: + commandName: createIndexes + - commandStartedEvent: + commandName: createIndexes + - commandFailedEvent: + commandName: createIndexes + - commandStartedEvent: + commandName: createIndexes + - commandFailedEvent: + commandName: createIndexes + - commandStartedEvent: + commandName: createIndexes + - commandSucceededEvent: + commandName: createIndexes + + - + description: 'collection.dropIndex retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + - + object: *utilCollection + name: createIndex + arguments: + keys: { x: 11 } + name: "x_11" + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [dropIndexes] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: dropIndex + arguments: + name: "x_11" + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandSucceededEvent: + commandName: dropIndexes + + - + description: 'collection.dropIndexes retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [dropIndexes] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: dropIndexes + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandSucceededEvent: + commandName: dropIndexes diff --git a/source/client-backpressure/tests/backpressure-retry-loop.yml.template b/source/client-backpressure/tests/backpressure-retry-loop.yml.template new file mode 100644 index 0000000000..049cbbac3a --- /dev/null +++ b/source/client-backpressure/tests/backpressure-retry-loop.yml.template @@ -0,0 +1,128 @@ +# Tests in this file are generated from backpressure-retry-loop.yml.template. + +description: tests that operations respect overload backoff retry loop + +schemaVersion: '1.0' + +runOnRequirements: + - + minServerVersion: '4.4' # failCommand + topologies: [replicaset, sharded, load-balanced] + +createEntities: + - + client: + id: &client client + useMultipleMongoses: false + observeEvents: [ 'commandStartedEvent', 'commandSucceededEvent', 'commandFailedEvent' ] + + - + client: + id: &failPointClient failPointClient + useMultipleMongoses: false + + - + database: + id: &utilDb utilDb + client: *failPointClient + databaseName: &database_name retryable-writes-tests + + - + collection: + id: &utilCollection utilCollection + database: *utilDb + collectionName: &collection_name coll + + - + database: + id: &database database + client: *client + databaseName: &database_name retryable-writes-tests + - + collection: + id: &collection collection + database: *database + collectionName: &collection_name coll + +initialData: + - + collectionName: *collection_name + databaseName: *database_name + documents: + - { _id: 1, x: 11 } + - { _id: 2, x: 22 } + +tests: +{% for operation in operations %} + - + description: '{{operation.object}}.{{operation.operation_name}} retries using operation loop' + operations: + - + object: *utilCollection + name: deleteMany + arguments: + filter: {} + + - + object: *utilCollection + name: deleteMany + arguments: + documents: + - { _id: 1, x: 11 } + + {%- if operation.operation_name == "dropIndex" %} + - + object: *utilCollection + name: createIndex + arguments: + keys: { x: 11 } + name: "x_11" + {%- endif %} + + + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [{{operation.command_name}}] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *{{operation.object}} + name: {{operation.operation_name}} + {%- if operation.arguments|length > 0 %} + arguments: + {%- for arg in operation.arguments %} + {{arg}} + {%- endfor -%} + {%- endif %} + {%- if operation.operation_name == "createChangeStream" %} + saveResultAsEntity: changeStream + {%- endif %} + expectError: false + + expectEvents: + - client: "client" + events: + - commandStartedEvent: + commandName: {{operation.command_name}} + - commandFailedEvent: + commandName: {{operation.command_name}} + - commandStartedEvent: + commandName: {{operation.command_name}} + - commandFailedEvent: + commandName: {{operation.command_name}} + - commandStartedEvent: + commandName: {{operation.command_name}} + - commandFailedEvent: + commandName: {{operation.command_name}} + - commandStartedEvent: + commandName: {{operation.command_name}} + - commandSucceededEvent: + commandName: {{operation.command_name}} +{% endfor -%} diff --git a/source/client-backpressure/tests/backpressure-retry-max-attempts.json b/source/client-backpressure/tests/backpressure-retry-max-attempts.json new file mode 100644 index 0000000000..36afc55cc5 --- /dev/null +++ b/source/client-backpressure/tests/backpressure-retry-max-attempts.json @@ -0,0 +1,1262 @@ +{ + "description": "tests that operations retry at most maxAttempts=5 times", + "schemaVersion": "1.0", + "runOnRequirements": [ + { + "minServerVersion": "4.4", + "topologies": [ + "replicaset", + "sharded", + "load-balanced" + ] + } + ], + "createEntities": [ + { + "client": { + "id": "client", + "useMultipleMongoses": false + } + }, + { + "client": { + "id": "failPointClient", + "useMultipleMongoses": false + } + }, + { + "database": { + "id": "database", + "client": "client", + "databaseName": "retryable-writes-tests" + } + }, + { + "collection": { + "id": "collection", + "database": "database", + "collectionName": "coll" + } + } + ], + "initialData": [ + { + "collectionName": "coll", + "databaseName": "retryable-writes-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + } + ] + } + ], + "tests": [ + { + "description": "client.listDatabases retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "listDatabases" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "client", + "name": "listDatabases", + "arguments": { + "filter": {} + }, + "expectError": true + } + ] + }, + { + "description": "client.listDatabaseNames retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "listDatabases" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "client", + "name": "listDatabaseNames", + "expectError": true + } + ] + }, + { + "description": "client.createChangeStream retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "client", + "name": "createChangeStream", + "arguments": { + "pipeline": [] + }, + "saveResultAsEntity": "changeStream", + "expectError": true + } + ] + }, + { + "description": "client.clientBulkWrite retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "bulkWrite" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "client", + "name": "clientBulkWrite", + "arguments": { + "models": [ + { + "insertOne": { + "namespace": "retryable-writes-tests.coll", + "document": { + "_id": 8, + "x": 88 + } + } + } + ] + }, + "expectError": true + } + ] + }, + { + "description": "database.aggregate retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "database", + "name": "aggregate", + "arguments": { + "pipeline": [ + { + "$listLocalSessions": {} + }, + { + "$limit": 1 + } + ] + }, + "expectError": true + } + ] + }, + { + "description": "database.listCollections retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "listCollections" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "database", + "name": "listCollections", + "arguments": { + "filter": {} + }, + "expectError": true + } + ] + }, + { + "description": "database.listCollectionNames retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "listCollections" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "database", + "name": "listCollectionNames", + "arguments": { + "filter": {} + }, + "expectError": true + } + ] + }, + { + "description": "database.runCommand retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "ping" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "database", + "name": "runCommand", + "arguments": { + "command": { + "ping": 1 + }, + "commandName": "ping" + }, + "expectError": true + } + ] + }, + { + "description": "database.createChangeStream retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "database", + "name": "createChangeStream", + "arguments": { + "pipeline": [] + }, + "saveResultAsEntity": "changeStream", + "expectError": true + } + ] + }, + { + "description": "collection.aggregate retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "aggregate", + "arguments": { + "pipeline": [] + }, + "expectError": true + } + ] + }, + { + "description": "collection.countDocuments retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "countDocuments", + "arguments": { + "filter": {} + }, + "expectError": true + } + ] + }, + { + "description": "collection.estimatedDocumentCount retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "count" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "estimatedDocumentCount", + "expectError": true + } + ] + }, + { + "description": "collection.distinct retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "distinct" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "distinct", + "arguments": { + "fieldName": "x", + "filter": {} + }, + "expectError": true + } + ] + }, + { + "description": "collection.find retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "find" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "find", + "arguments": { + "filter": {} + }, + "expectError": true + } + ] + }, + { + "description": "collection.findOne retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "find" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "findOne", + "arguments": { + "filter": {} + }, + "expectError": true + } + ] + }, + { + "description": "collection.listIndexes retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "listIndexes" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "listIndexes", + "expectError": true + } + ] + }, + { + "description": "collection.listIndexNames retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "listIndexes" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "listIndexNames", + "expectError": true + } + ] + }, + { + "description": "collection.createChangeStream retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "createChangeStream", + "arguments": { + "pipeline": [] + }, + "saveResultAsEntity": "changeStream", + "expectError": true + } + ] + }, + { + "description": "collection.insertOne retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "insert" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "insertOne", + "arguments": { + "document": { + "_id": 2, + "x": 22 + } + }, + "expectError": true + } + ] + }, + { + "description": "collection.insertMany retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "insert" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "insertMany", + "arguments": { + "documents": [ + { + "_id": 2, + "x": 22 + } + ] + }, + "expectError": true + } + ] + }, + { + "description": "collection.deleteOne retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "delete" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "deleteOne", + "arguments": { + "filter": {} + }, + "expectError": true + } + ] + }, + { + "description": "collection.deleteMany retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "delete" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "deleteMany", + "arguments": { + "filter": {} + }, + "expectError": true + } + ] + }, + { + "description": "collection.replaceOne retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "update" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "replaceOne", + "arguments": { + "filter": {}, + "replacement": { + "x": 22 + } + }, + "expectError": true + } + ] + }, + { + "description": "collection.updateOne retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "update" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "updateOne", + "arguments": { + "filter": {}, + "update": { + "$set": { + "x": 22 + } + } + }, + "expectError": true + } + ] + }, + { + "description": "collection.updateMany retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "update" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "updateMany", + "arguments": { + "filter": {}, + "update": { + "$set": { + "x": 22 + } + } + }, + "expectError": true + } + ] + }, + { + "description": "collection.findOneAndDelete retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "findAndModify" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "findOneAndDelete", + "arguments": { + "filter": {} + }, + "expectError": true + } + ] + }, + { + "description": "collection.findOneAndReplace retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "findAndModify" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "findOneAndReplace", + "arguments": { + "filter": {}, + "replacement": { + "x": 22 + } + }, + "expectError": true + } + ] + }, + { + "description": "collection.findOneAndUpdate retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "findAndModify" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "findOneAndUpdate", + "arguments": { + "filter": {}, + "update": { + "$set": { + "x": 22 + } + } + }, + "expectError": true + } + ] + }, + { + "description": "collection.bulkWrite retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "insert" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "bulkWrite", + "arguments": { + "requests": [ + { + "insertOne": { + "document": { + "_id": 2, + "x": 22 + } + } + } + ] + }, + "expectError": true + } + ] + }, + { + "description": "collection.createIndex retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "createIndexes" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "createIndex", + "arguments": { + "keys": { + "x": 11 + }, + "name": "x_11" + }, + "expectError": true + } + ] + }, + { + "description": "collection.dropIndex retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "dropIndexes" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "dropIndex", + "arguments": { + "name": "x_11" + }, + "expectError": true + } + ] + }, + { + "description": "collection.dropIndexes retries at most maxAttempts times (maxAttempts=5)", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 6 + }, + "data": { + "failCommands": [ + "dropIndexes" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "dropIndexes", + "expectError": true + } + ] + } + ] +} diff --git a/source/client-backpressure/tests/backpressure-retry-max-attempts.yml b/source/client-backpressure/tests/backpressure-retry-max-attempts.yml new file mode 100644 index 0000000000..998b48c78e --- /dev/null +++ b/source/client-backpressure/tests/backpressure-retry-max-attempts.yml @@ -0,0 +1,753 @@ +# Tests in this file are generated from backpressure-retry-loop.yml.template. + +description: tests that operations retry at most maxAttempts=5 times + +schemaVersion: '1.0' + +runOnRequirements: + - + minServerVersion: '4.4' # failCommand + topologies: [replicaset, sharded, load-balanced] + +createEntities: + - + client: + id: &client client + useMultipleMongoses: false + + - + client: + id: &failPointClient failPointClient + useMultipleMongoses: false + + - + database: + id: &database database + client: *client + databaseName: &database_name retryable-writes-tests + - + collection: + id: &collection collection + database: *database + collectionName: &collection_name coll + +initialData: + - + collectionName: *collection_name + databaseName: *database_name + documents: + - { _id: 1, x: 11 } + - { _id: 2, x: 22 } + +tests: + + - + description: 'client.listDatabases retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [listDatabases] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *client + name: listDatabases + arguments: + filter: {} + expectError: true + + - + description: 'client.listDatabaseNames retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [listDatabases] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *client + name: listDatabaseNames + expectError: true + + - + description: 'client.createChangeStream retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [aggregate] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *client + name: createChangeStream + arguments: + pipeline: [] + saveResultAsEntity: changeStream + expectError: true + + - + description: 'client.clientBulkWrite retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [bulkWrite] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *client + name: clientBulkWrite + arguments: + models: + - insertOne: + namespace: retryable-writes-tests.coll + document: { _id: 8, x: 88 } + expectError: true + + - + description: 'database.aggregate retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [aggregate] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *database + name: aggregate + arguments: + pipeline: [ { $listLocalSessions: {} }, { $limit: 1 } ] + expectError: true + + - + description: 'database.listCollections retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [listCollections] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *database + name: listCollections + arguments: + filter: {} + expectError: true + + - + description: 'database.listCollectionNames retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [listCollections] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *database + name: listCollectionNames + arguments: + filter: {} + expectError: true + + - + description: 'database.runCommand retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [ping] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *database + name: runCommand + arguments: + command: { ping: 1 } + commandName: ping + expectError: true + + - + description: 'database.createChangeStream retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [aggregate] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *database + name: createChangeStream + arguments: + pipeline: [] + saveResultAsEntity: changeStream + expectError: true + + - + description: 'collection.aggregate retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [aggregate] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: aggregate + arguments: + pipeline: [] + expectError: true + + - + description: 'collection.countDocuments retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [aggregate] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: countDocuments + arguments: + filter: {} + expectError: true + + - + description: 'collection.estimatedDocumentCount retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [count] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: estimatedDocumentCount + expectError: true + + - + description: 'collection.distinct retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [distinct] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: distinct + arguments: + fieldName: x + filter: {} + expectError: true + + - + description: 'collection.find retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [find] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: find + arguments: + filter: {} + expectError: true + + - + description: 'collection.findOne retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [find] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: findOne + arguments: + filter: {} + expectError: true + + - + description: 'collection.listIndexes retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [listIndexes] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: listIndexes + expectError: true + + - + description: 'collection.listIndexNames retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [listIndexes] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: listIndexNames + expectError: true + + - + description: 'collection.createChangeStream retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [aggregate] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: createChangeStream + arguments: + pipeline: [] + saveResultAsEntity: changeStream + expectError: true + + - + description: 'collection.insertOne retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [insert] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: insertOne + arguments: + document: { _id: 2, x: 22 } + expectError: true + + - + description: 'collection.insertMany retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [insert] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: insertMany + arguments: + documents: + - { _id: 2, x: 22 } + expectError: true + + - + description: 'collection.deleteOne retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [delete] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: deleteOne + arguments: + filter: {} + expectError: true + + - + description: 'collection.deleteMany retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [delete] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: deleteMany + arguments: + filter: {} + expectError: true + + - + description: 'collection.replaceOne retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [update] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: replaceOne + arguments: + filter: {} + replacement: { x: 22 } + expectError: true + + - + description: 'collection.updateOne retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [update] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: updateOne + arguments: + filter: {} + update: { $set: { x: 22 } } + expectError: true + + - + description: 'collection.updateMany retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [update] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: updateMany + arguments: + filter: {} + update: { $set: { x: 22 } } + expectError: true + + - + description: 'collection.findOneAndDelete retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [findAndModify] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: findOneAndDelete + arguments: + filter: {} + expectError: true + + - + description: 'collection.findOneAndReplace retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [findAndModify] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: findOneAndReplace + arguments: + filter: {} + replacement: { x: 22 } + expectError: true + + - + description: 'collection.findOneAndUpdate retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [findAndModify] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: findOneAndUpdate + arguments: + filter: {} + update: { $set: { x: 22 } } + expectError: true + + - + description: 'collection.bulkWrite retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [insert] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: bulkWrite + arguments: + requests: + - insertOne: + document: { _id: 2, x: 22 } + expectError: true + + - + description: 'collection.createIndex retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [createIndexes] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: createIndex + arguments: + keys: { x: 11 } + name: "x_11" + expectError: true + + - + description: 'collection.dropIndex retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [dropIndexes] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: dropIndex + arguments: + name: "x_11" + expectError: true + + - + description: 'collection.dropIndexes retries at most maxAttempts times (maxAttempts=5)' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [dropIndexes] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *collection + name: dropIndexes + expectError: true diff --git a/source/client-backpressure/tests/backpressure-retry-max-attempts.yml.template b/source/client-backpressure/tests/backpressure-retry-max-attempts.yml.template new file mode 100644 index 0000000000..bf089211fd --- /dev/null +++ b/source/client-backpressure/tests/backpressure-retry-max-attempts.yml.template @@ -0,0 +1,72 @@ +# Tests in this file are generated from backpressure-retry-max-attempts.yml.template. + +description: tests that operations retry at most maxAttempts=5 times + +schemaVersion: '1.0' + +runOnRequirements: + - + minServerVersion: '4.4' # failCommand + topologies: [replicaset, sharded, load-balanced] + +createEntities: + - + client: + id: &client client + useMultipleMongoses: false + + - + client: + id: &failPointClient failPointClient + useMultipleMongoses: false + + - + database: + id: &database database + client: *client + databaseName: &database_name retryable-writes-tests + - + collection: + id: &collection collection + database: *database + collectionName: &collection_name coll + +initialData: + - + collectionName: *collection_name + databaseName: *database_name + documents: + - { _id: 1, x: 11 } + - { _id: 2, x: 22 } + +tests: +{% for operation in operations %} + - + description: '{{operation.object}}.{{operation.operation_name}} retries at most maxAttempts=5 times' + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 6 } + data: + failCommands: [{{operation.command_name}}] + errorLabels: ["RetryableError", "SystemOverloadedError"] + errorCode: 2 + + - + object: *{{operation.object}} + name: {{operation.operation_name}} + {%- if operation.arguments|length > 0 %} + arguments: + {%- for arg in operation.arguments %} + {{arg}} + {%- endfor -%} + {%- endif %} + {%- if operation.operation_name == "createChangeStream" %} + saveResultAsEntity: changeStream + {%- endif %} + expectError: true +{% endfor -%} diff --git a/source/etc/generate-backpressure-retryability-tests.py b/source/etc/generate-backpressure-retryability-tests.py new file mode 100644 index 0000000000..305cfa585d --- /dev/null +++ b/source/etc/generate-backpressure-retryability-tests.py @@ -0,0 +1,125 @@ +from collections import namedtuple +from jinja2 import Template +import os +import sys + +Operation = namedtuple( + 'Operation', ['operation_name', 'command_name', 'object', 'arguments']) + +CLIENT_BULK_WRITE_ARGUMENTS = '''models: + - insertOne: + namespace: retryable-writes-tests.coll + document: { _id: 8, x: 88 }''' + +CLIENT_OPERATIONS = [ + Operation('listDatabases', 'listDatabases', 'client', ['filter: {}']), + Operation('listDatabaseNames', 'listDatabases', 'client', []), + Operation('createChangeStream', 'aggregate', 'client', ['pipeline: []']), + Operation('clientBulkWrite', 'bulkWrite', 'client', [CLIENT_BULK_WRITE_ARGUMENTS]) +] + +RUN_COMMAND_ARGUMENTS = '''command: { ping: 1 } + commandName: ping''' + +DB_OPERATIONS = [ + Operation('aggregate', 'aggregate', 'database', [ + 'pipeline: [ { $listLocalSessions: {} }, { $limit: 1 } ]']), + Operation('listCollections', 'listCollections', + 'database', ['filter: {}']), + Operation('listCollectionNames', 'listCollections', + 'database', ['filter: {}']), # Optional. + Operation('runCommand', 'ping', 'database', [RUN_COMMAND_ARGUMENTS]), + Operation('createChangeStream', 'aggregate', 'database', ['pipeline: []']) +] + +INSERT_MANY_ARGUMENTS = '''documents: + - { _id: 2, x: 22 }''' + +BULK_WRITE_ARGUMENTS = '''requests: + - insertOne: + document: { _id: 2, x: 22 }''' + +COLLECTION_READ_OPERATIONS = [ + Operation('aggregate', 'aggregate', 'collection', ['pipeline: []']), + # Operation('count', 'count', 'collection', ['filter: {}']), # Deprecated. + Operation('countDocuments', 'aggregate', 'collection', ['filter: {}']), + Operation('estimatedDocumentCount', 'count', 'collection', []), + Operation('distinct', 'distinct', 'collection', + ['fieldName: x', 'filter: {}']), + Operation('find', 'find', 'collection', ['filter: {}']), + Operation('findOne', 'find', 'collection', ['filter: {}']), # Optional. + Operation('listIndexes', 'listIndexes', 'collection', []), + Operation('listIndexNames', 'listIndexes', 'collection', []), # Optional. + Operation('createChangeStream', 'aggregate', + 'collection', ['pipeline: []']), +] + +COLLECTION_WRITE_OPERATIONS = [ + Operation('insertOne', 'insert', 'collection', + ['document: { _id: 2, x: 22 }']), + Operation('insertMany', 'insert', 'collection', [INSERT_MANY_ARGUMENTS]), + Operation('deleteOne', 'delete', 'collection', ['filter: {}']), + Operation('deleteMany', 'delete', 'collection', ['filter: {}']), + Operation('replaceOne', 'update', 'collection', [ + 'filter: {}', 'replacement: { x: 22 }']), + Operation('updateOne', 'update', 'collection', [ + 'filter: {}', 'update: { $set: { x: 22 } }']), + Operation('updateMany', 'update', 'collection', [ + 'filter: {}', 'update: { $set: { x: 22 } }']), + Operation('findOneAndDelete', 'findAndModify', + 'collection', ['filter: {}']), + Operation('findOneAndReplace', 'findAndModify', 'collection', + ['filter: {}', 'replacement: { x: 22 }']), + Operation('findOneAndUpdate', 'findAndModify', 'collection', + ['filter: {}', 'update: { $set: { x: 22 } }']), + Operation('bulkWrite', 'insert', 'collection', [BULK_WRITE_ARGUMENTS]), + Operation('createIndex', 'createIndexes', 'collection', + ['keys: { x: 11 }', 'name: "x_11"']), + Operation('dropIndex', 'dropIndexes', 'collection', ['name: "x_11"']), + Operation('dropIndexes', 'dropIndexes', 'collection', []), +] + +COLLECTION_OPERATIONS = COLLECTION_READ_OPERATIONS + COLLECTION_WRITE_OPERATIONS + +# Session and GridFS operations are generally tested in other files, so they're not included in the list of all +# operations. Individual generation functions can choose to include them if needed. +OPERATIONS = CLIENT_OPERATIONS + DB_OPERATIONS + COLLECTION_OPERATIONS + +# ./source/etc +DIR = os.path.dirname(os.path.realpath(__file__)) + + +def get_template(file, templates_dir): + path = f'{templates_dir}/{file}.yml.template' + return Template(open(path, 'r').read()) + + +def write_yaml(file, template, tests_dir, injections): + rendered = template.render(**injections) + path = f'{tests_dir}/{file}.yml' + open(path, 'w').write(rendered) + + +def generate(name, templates_dir, tests_dir, operations): + template = get_template(name, templates_dir) + injections = { + 'operations': operations, + } + write_yaml(name, template, tests_dir, injections) + + +def generate_retry_loop_tests(): + templates_dir = f'{os.path.dirname(DIR)}/client-backpressure/tests' + tests_dir = f'{os.path.dirname(DIR)}/client-backpressure/tests' + generate('backpressure-retry-loop', templates_dir, + tests_dir, OPERATIONS) + + +def generate_max_attempts_tests(): + templates_dir = f'{os.path.dirname(DIR)}/client-backpressure/tests' + tests_dir = f'{os.path.dirname(DIR)}/client-backpressure/tests' + generate('backpressure-retry-max-attempts', templates_dir, + tests_dir, OPERATIONS) + +generate_retry_loop_tests() +generate_max_attempts_tests() \ No newline at end of file From 072b45371972be8d136753ae311a039e3c42d9f0 Mon Sep 17 00:00:00 2001 From: bailey Date: Mon, 1 Dec 2025 16:39:22 -0700 Subject: [PATCH 05/19] test fixes and add prose test - add prose test - add assertions on the number of retries for maxAttempts tests - don't run clientBulkWrite tests on <8.0 servers --- source/client-backpressure/tests/README.md | 46 + .../tests/backpressure-retry-loop.json | 3 + .../tests/backpressure-retry-loop.yml | 2 + .../backpressure-retry-loop.yml.template | 4 + .../backpressure-retry-max-attempts.json | 2408 +++++++++++++++-- .../tests/backpressure-retry-max-attempts.yml | 1125 +++++++- ...ckpressure-retry-max-attempts.yml.template | 38 +- 7 files changed, 3400 insertions(+), 226 deletions(-) diff --git a/source/client-backpressure/tests/README.md b/source/client-backpressure/tests/README.md index 7a70e4ad76..a4e62b9ec1 100644 --- a/source/client-backpressure/tests/README.md +++ b/source/client-backpressure/tests/README.md @@ -10,6 +10,52 @@ retryable reads. These tests utilize the [Unified Test Format](../../unified-tes Several prose tests, which are not easily expressed in YAML, are also presented in this file. Those tests will need to be manually implemented by each driver. +### Prose Tests + +#### Test 1: Operation Retry Uses Exponential Backoff + +Drivers should test that retries do not occur immediately when a SystemOverloadedError is encountered. + +1. let `client` be a `MongoClient` +2. let `collection` be a collection +3. Now, run transactions without backoff: + 1. Configure the random number generator used for jitter to always return `0` -- this effectively disables backoff. + + 2. Configure the following failPoint: + + ```javascript + { + configureFailPoint: 'failCommand', + mode: 'alwaysOn', + data: { + failCommands: ['insert'], + errorCode: 2, + errorLabels: ['SystemOverloadedError', 'RetryableError'] + } + } + ``` + + 3. Execute the following command. Expect that the command errors. Measure the duration of the command execution. + + ```javascript + const start = performance.now(); + expect( + await coll.insertOne({ a: 1 }).catch(e => e) + ).to.be.an.instanceof(MongoServerError); + const end = performance.now(); + ``` + + 4. Configure the random number generator used for jitter to always return `1`. + + 5. Execute step 3 again. + + 6. Compare the two time between the two runs. + ```python + assertTrue(absolute_value(with_backoff_time - (no_backoff_time + 3.1 seconds)) < 1) + ``` + The sum of 5 backoffs is 3.1 seconds. There is a 1-second window to account for potential variance between the two + runs. + ## Changelog - 2025-XX-XX: Initial version. diff --git a/source/client-backpressure/tests/backpressure-retry-loop.json b/source/client-backpressure/tests/backpressure-retry-loop.json index 21fc802344..ae944f73ad 100644 --- a/source/client-backpressure/tests/backpressure-retry-loop.json +++ b/source/client-backpressure/tests/backpressure-retry-loop.json @@ -381,6 +381,9 @@ }, { "description": "client.clientBulkWrite retries using operation loop", + "runOnRequirements": { + "minServerVersion": "8.0" + }, "operations": [ { "object": "utilCollection", diff --git a/source/client-backpressure/tests/backpressure-retry-loop.yml b/source/client-backpressure/tests/backpressure-retry-loop.yml index ec2fa25b8f..c612986233 100644 --- a/source/client-backpressure/tests/backpressure-retry-loop.yml +++ b/source/client-backpressure/tests/backpressure-retry-loop.yml @@ -223,6 +223,8 @@ tests: - description: 'client.clientBulkWrite retries using operation loop' + runOnRequirements: + minServerVersion: '8.0' operations: - object: *utilCollection diff --git a/source/client-backpressure/tests/backpressure-retry-loop.yml.template b/source/client-backpressure/tests/backpressure-retry-loop.yml.template index 049cbbac3a..cad83625d3 100644 --- a/source/client-backpressure/tests/backpressure-retry-loop.yml.template +++ b/source/client-backpressure/tests/backpressure-retry-loop.yml.template @@ -56,6 +56,10 @@ tests: {% for operation in operations %} - description: '{{operation.object}}.{{operation.operation_name}} retries using operation loop' + {%- if ((operation.operation_name == 'clientBulkWrite')) %} + runOnRequirements: + minServerVersion: '8.0' + {%- endif %} operations: - object: *utilCollection diff --git a/source/client-backpressure/tests/backpressure-retry-max-attempts.json b/source/client-backpressure/tests/backpressure-retry-max-attempts.json index 36afc55cc5..7e9cc67d7a 100644 --- a/source/client-backpressure/tests/backpressure-retry-max-attempts.json +++ b/source/client-backpressure/tests/backpressure-retry-max-attempts.json @@ -15,7 +15,12 @@ { "client": { "id": "client", - "useMultipleMongoses": false + "useMultipleMongoses": false, + "observeEvents": [ + "commandStartedEvent", + "commandSucceededEvent", + "commandFailedEvent" + ] } }, { @@ -57,7 +62,7 @@ ], "tests": [ { - "description": "client.listDatabases retries at most maxAttempts times (maxAttempts=5)", + "description": "client.listDatabases retries at most maxAttempts=5 times", "operations": [ { "name": "failPoint", @@ -66,9 +71,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "listDatabases" @@ -90,10 +93,77 @@ }, "expectError": true } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + } + ] + } ] }, { - "description": "client.listDatabaseNames retries at most maxAttempts times (maxAttempts=5)", + "description": "client.listDatabaseNames retries at most maxAttempts=5 times", "operations": [ { "name": "failPoint", @@ -102,9 +172,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "listDatabases" @@ -123,10 +191,77 @@ "name": "listDatabaseNames", "expectError": true } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + } + ] + } ] }, { - "description": "client.createChangeStream retries at most maxAttempts times (maxAttempts=5)", + "description": "client.createChangeStream retries at most maxAttempts=5 times", "operations": [ { "name": "failPoint", @@ -135,9 +270,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "aggregate" @@ -160,10 +293,80 @@ "saveResultAsEntity": "changeStream", "expectError": true } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + } + ] + } ] }, { - "description": "client.clientBulkWrite retries at most maxAttempts times (maxAttempts=5)", + "description": "client.clientBulkWrite retries at most maxAttempts=5 times", + "runOnRequirements": { + "minServerVersion": "8.0" + }, "operations": [ { "name": "failPoint", @@ -172,9 +375,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "bulkWrite" @@ -206,10 +407,77 @@ }, "expectError": true } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandFailedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandStartedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandFailedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandStartedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandFailedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandStartedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandFailedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandStartedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandFailedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandStartedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandFailedEvent": { + "commandName": "bulkWrite" + } + } + ] + } ] }, { - "description": "database.aggregate retries at most maxAttempts times (maxAttempts=5)", + "description": "database.aggregate retries at most maxAttempts=5 times", "operations": [ { "name": "failPoint", @@ -218,9 +486,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "aggregate" @@ -249,10 +515,77 @@ }, "expectError": true } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + } + ] + } ] }, { - "description": "database.listCollections retries at most maxAttempts times (maxAttempts=5)", + "description": "database.listCollections retries at most maxAttempts=5 times", "operations": [ { "name": "failPoint", @@ -261,9 +594,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "listCollections" @@ -285,10 +616,77 @@ }, "expectError": true } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + } + ] + } ] }, { - "description": "database.listCollectionNames retries at most maxAttempts times (maxAttempts=5)", + "description": "database.listCollectionNames retries at most maxAttempts=5 times", "operations": [ { "name": "failPoint", @@ -297,9 +695,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "listCollections" @@ -321,10 +717,77 @@ }, "expectError": true } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + } + ] + } ] }, { - "description": "database.runCommand retries at most maxAttempts times (maxAttempts=5)", + "description": "database.runCommand retries at most maxAttempts=5 times", "operations": [ { "name": "failPoint", @@ -333,9 +796,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "ping" @@ -360,10 +821,77 @@ }, "expectError": true } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "ping" + } + }, + { + "commandFailedEvent": { + "commandName": "ping" + } + }, + { + "commandStartedEvent": { + "commandName": "ping" + } + }, + { + "commandFailedEvent": { + "commandName": "ping" + } + }, + { + "commandStartedEvent": { + "commandName": "ping" + } + }, + { + "commandFailedEvent": { + "commandName": "ping" + } + }, + { + "commandStartedEvent": { + "commandName": "ping" + } + }, + { + "commandFailedEvent": { + "commandName": "ping" + } + }, + { + "commandStartedEvent": { + "commandName": "ping" + } + }, + { + "commandFailedEvent": { + "commandName": "ping" + } + }, + { + "commandStartedEvent": { + "commandName": "ping" + } + }, + { + "commandFailedEvent": { + "commandName": "ping" + } + } + ] + } ] }, { - "description": "database.createChangeStream retries at most maxAttempts times (maxAttempts=5)", + "description": "database.createChangeStream retries at most maxAttempts=5 times", "operations": [ { "name": "failPoint", @@ -372,9 +900,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "aggregate" @@ -397,10 +923,77 @@ "saveResultAsEntity": "changeStream", "expectError": true } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + } + ] + } ] }, { - "description": "collection.aggregate retries at most maxAttempts times (maxAttempts=5)", + "description": "collection.aggregate retries at most maxAttempts=5 times", "operations": [ { "name": "failPoint", @@ -409,9 +1002,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "aggregate" @@ -433,29 +1024,94 @@ }, "expectError": true } - ] - }, - { - "description": "collection.countDocuments retries at most maxAttempts times (maxAttempts=5)", - "operations": [ - { - "name": "failPoint", - "object": "testRunner", - "arguments": { - "client": "failPointClient", - "failPoint": { - "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, - "data": { - "failCommands": [ - "aggregate" - ], - "errorLabels": [ - "RetryableError", - "SystemOverloadedError" - ], + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + } + ] + } + ] + }, + { + "description": "collection.countDocuments retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], "errorCode": 2 } } @@ -469,10 +1125,77 @@ }, "expectError": true } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + } + ] + } ] }, { - "description": "collection.estimatedDocumentCount retries at most maxAttempts times (maxAttempts=5)", + "description": "collection.estimatedDocumentCount retries at most maxAttempts=5 times", "operations": [ { "name": "failPoint", @@ -481,9 +1204,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "count" @@ -502,10 +1223,77 @@ "name": "estimatedDocumentCount", "expectError": true } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "count" + } + }, + { + "commandFailedEvent": { + "commandName": "count" + } + }, + { + "commandStartedEvent": { + "commandName": "count" + } + }, + { + "commandFailedEvent": { + "commandName": "count" + } + }, + { + "commandStartedEvent": { + "commandName": "count" + } + }, + { + "commandFailedEvent": { + "commandName": "count" + } + }, + { + "commandStartedEvent": { + "commandName": "count" + } + }, + { + "commandFailedEvent": { + "commandName": "count" + } + }, + { + "commandStartedEvent": { + "commandName": "count" + } + }, + { + "commandFailedEvent": { + "commandName": "count" + } + }, + { + "commandStartedEvent": { + "commandName": "count" + } + }, + { + "commandFailedEvent": { + "commandName": "count" + } + } + ] + } ] }, { - "description": "collection.distinct retries at most maxAttempts times (maxAttempts=5)", + "description": "collection.distinct retries at most maxAttempts=5 times", "operations": [ { "name": "failPoint", @@ -514,9 +1302,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "distinct" @@ -539,10 +1325,77 @@ }, "expectError": true } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "distinct" + } + }, + { + "commandFailedEvent": { + "commandName": "distinct" + } + }, + { + "commandStartedEvent": { + "commandName": "distinct" + } + }, + { + "commandFailedEvent": { + "commandName": "distinct" + } + }, + { + "commandStartedEvent": { + "commandName": "distinct" + } + }, + { + "commandFailedEvent": { + "commandName": "distinct" + } + }, + { + "commandStartedEvent": { + "commandName": "distinct" + } + }, + { + "commandFailedEvent": { + "commandName": "distinct" + } + }, + { + "commandStartedEvent": { + "commandName": "distinct" + } + }, + { + "commandFailedEvent": { + "commandName": "distinct" + } + }, + { + "commandStartedEvent": { + "commandName": "distinct" + } + }, + { + "commandFailedEvent": { + "commandName": "distinct" + } + } + ] + } ] }, { - "description": "collection.find retries at most maxAttempts times (maxAttempts=5)", + "description": "collection.find retries at most maxAttempts=5 times", "operations": [ { "name": "failPoint", @@ -551,9 +1404,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "find" @@ -575,10 +1426,77 @@ }, "expectError": true } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + } + ] + } ] }, { - "description": "collection.findOne retries at most maxAttempts times (maxAttempts=5)", + "description": "collection.findOne retries at most maxAttempts=5 times", "operations": [ { "name": "failPoint", @@ -587,9 +1505,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "find" @@ -611,10 +1527,77 @@ }, "expectError": true } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + } + ] + } ] }, { - "description": "collection.listIndexes retries at most maxAttempts times (maxAttempts=5)", + "description": "collection.listIndexes retries at most maxAttempts=5 times", "operations": [ { "name": "failPoint", @@ -623,9 +1606,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "listIndexes" @@ -644,10 +1625,77 @@ "name": "listIndexes", "expectError": true } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + } + ] + } ] }, { - "description": "collection.listIndexNames retries at most maxAttempts times (maxAttempts=5)", + "description": "collection.listIndexNames retries at most maxAttempts=5 times", "operations": [ { "name": "failPoint", @@ -656,9 +1704,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "listIndexes" @@ -677,10 +1723,77 @@ "name": "listIndexNames", "expectError": true } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + } + ] + } ] }, { - "description": "collection.createChangeStream retries at most maxAttempts times (maxAttempts=5)", + "description": "collection.createChangeStream retries at most maxAttempts=5 times", "operations": [ { "name": "failPoint", @@ -689,9 +1802,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "aggregate" @@ -714,10 +1825,77 @@ "saveResultAsEntity": "changeStream", "expectError": true } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + } + ] + } ] }, { - "description": "collection.insertOne retries at most maxAttempts times (maxAttempts=5)", + "description": "collection.insertOne retries at most maxAttempts=5 times", "operations": [ { "name": "failPoint", @@ -726,9 +1904,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "insert" @@ -753,10 +1929,77 @@ }, "expectError": true } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + } + ] + } ] }, { - "description": "collection.insertMany retries at most maxAttempts times (maxAttempts=5)", + "description": "collection.insertMany retries at most maxAttempts=5 times", "operations": [ { "name": "failPoint", @@ -765,9 +2008,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "insert" @@ -794,10 +2035,77 @@ }, "expectError": true } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + } + ] + } ] }, { - "description": "collection.deleteOne retries at most maxAttempts times (maxAttempts=5)", + "description": "collection.deleteOne retries at most maxAttempts=5 times", "operations": [ { "name": "failPoint", @@ -806,9 +2114,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "delete" @@ -830,10 +2136,77 @@ }, "expectError": true } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + } + ] + } ] }, { - "description": "collection.deleteMany retries at most maxAttempts times (maxAttempts=5)", + "description": "collection.deleteMany retries at most maxAttempts=5 times", "operations": [ { "name": "failPoint", @@ -842,9 +2215,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "delete" @@ -866,10 +2237,77 @@ }, "expectError": true } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + } + ] + } ] }, { - "description": "collection.replaceOne retries at most maxAttempts times (maxAttempts=5)", + "description": "collection.replaceOne retries at most maxAttempts=5 times", "operations": [ { "name": "failPoint", @@ -878,9 +2316,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "update" @@ -892,23 +2328,90 @@ "errorCode": 2 } } - } - }, - { - "object": "collection", - "name": "replaceOne", - "arguments": { - "filter": {}, - "replacement": { - "x": 22 - } - }, - "expectError": true + } + }, + { + "object": "collection", + "name": "replaceOne", + "arguments": { + "filter": {}, + "replacement": { + "x": 22 + } + }, + "expectError": true + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + } + ] } ] }, { - "description": "collection.updateOne retries at most maxAttempts times (maxAttempts=5)", + "description": "collection.updateOne retries at most maxAttempts=5 times", "operations": [ { "name": "failPoint", @@ -917,9 +2420,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "update" @@ -946,10 +2447,77 @@ }, "expectError": true } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + } + ] + } ] }, { - "description": "collection.updateMany retries at most maxAttempts times (maxAttempts=5)", + "description": "collection.updateMany retries at most maxAttempts=5 times", "operations": [ { "name": "failPoint", @@ -958,9 +2526,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "update" @@ -987,10 +2553,77 @@ }, "expectError": true } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + } + ] + } ] }, { - "description": "collection.findOneAndDelete retries at most maxAttempts times (maxAttempts=5)", + "description": "collection.findOneAndDelete retries at most maxAttempts=5 times", "operations": [ { "name": "failPoint", @@ -999,9 +2632,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "findAndModify" @@ -1023,10 +2654,77 @@ }, "expectError": true } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + } + ] + } ] }, { - "description": "collection.findOneAndReplace retries at most maxAttempts times (maxAttempts=5)", + "description": "collection.findOneAndReplace retries at most maxAttempts=5 times", "operations": [ { "name": "failPoint", @@ -1035,9 +2733,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "findAndModify" @@ -1062,10 +2758,77 @@ }, "expectError": true } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + } + ] + } ] }, { - "description": "collection.findOneAndUpdate retries at most maxAttempts times (maxAttempts=5)", + "description": "collection.findOneAndUpdate retries at most maxAttempts=5 times", "operations": [ { "name": "failPoint", @@ -1074,9 +2837,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "findAndModify" @@ -1103,10 +2864,77 @@ }, "expectError": true } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + } + ] + } ] }, { - "description": "collection.bulkWrite retries at most maxAttempts times (maxAttempts=5)", + "description": "collection.bulkWrite retries at most maxAttempts=5 times", "operations": [ { "name": "failPoint", @@ -1115,9 +2943,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "insert" @@ -1148,10 +2974,77 @@ }, "expectError": true } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + } + ] + } ] }, { - "description": "collection.createIndex retries at most maxAttempts times (maxAttempts=5)", + "description": "collection.createIndex retries at most maxAttempts=5 times", "operations": [ { "name": "failPoint", @@ -1160,9 +3053,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "createIndexes" @@ -1187,10 +3078,77 @@ }, "expectError": true } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "createIndexes" + } + } + ] + } ] }, { - "description": "collection.dropIndex retries at most maxAttempts times (maxAttempts=5)", + "description": "collection.dropIndex retries at most maxAttempts=5 times", "operations": [ { "name": "failPoint", @@ -1199,9 +3157,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "dropIndexes" @@ -1223,10 +3179,77 @@ }, "expectError": true } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + } + ] + } ] }, { - "description": "collection.dropIndexes retries at most maxAttempts times (maxAttempts=5)", + "description": "collection.dropIndexes retries at most maxAttempts=5 times", "operations": [ { "name": "failPoint", @@ -1235,9 +3258,7 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": { - "times": 6 - }, + "mode": "alwaysOn", "data": { "failCommands": [ "dropIndexes" @@ -1256,6 +3277,73 @@ "name": "dropIndexes", "expectError": true } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + } + ] + } ] } ] diff --git a/source/client-backpressure/tests/backpressure-retry-max-attempts.yml b/source/client-backpressure/tests/backpressure-retry-max-attempts.yml index 998b48c78e..9161828e99 100644 --- a/source/client-backpressure/tests/backpressure-retry-max-attempts.yml +++ b/source/client-backpressure/tests/backpressure-retry-max-attempts.yml @@ -1,4 +1,4 @@ -# Tests in this file are generated from backpressure-retry-loop.yml.template. +# Tests in this file are generated from backpressure-retry-max-attempts.yml.template. description: tests that operations retry at most maxAttempts=5 times @@ -14,6 +14,7 @@ createEntities: client: id: &client client useMultipleMongoses: false + observeEvents: [ 'commandStartedEvent', 'commandSucceededEvent', 'commandFailedEvent' ] - client: @@ -42,7 +43,7 @@ initialData: tests: - - description: 'client.listDatabases retries at most maxAttempts times (maxAttempts=5)' + description: 'client.listDatabases retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -50,7 +51,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [listDatabases] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -63,8 +64,39 @@ tests: filter: {} expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + + - - description: 'client.listDatabaseNames retries at most maxAttempts times (maxAttempts=5)' + description: 'client.listDatabaseNames retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -72,7 +104,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [listDatabases] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -83,8 +115,39 @@ tests: name: listDatabaseNames expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + + - - description: 'client.createChangeStream retries at most maxAttempts times (maxAttempts=5)' + description: 'client.createChangeStream retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -92,7 +155,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [aggregate] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -106,8 +169,41 @@ tests: saveResultAsEntity: changeStream expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + + - - description: 'client.clientBulkWrite retries at most maxAttempts times (maxAttempts=5)' + description: 'client.clientBulkWrite retries at most maxAttempts=5 times' + runOnRequirements: + minServerVersion: '8.0' operations: - name: failPoint object: testRunner @@ -115,7 +211,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [bulkWrite] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -131,8 +227,39 @@ tests: document: { _id: 8, x: 88 } expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: bulkWrite + - commandFailedEvent: + commandName: bulkWrite + - commandStartedEvent: + commandName: bulkWrite + - commandFailedEvent: + commandName: bulkWrite + - commandStartedEvent: + commandName: bulkWrite + - commandFailedEvent: + commandName: bulkWrite + - commandStartedEvent: + commandName: bulkWrite + - commandFailedEvent: + commandName: bulkWrite + - commandStartedEvent: + commandName: bulkWrite + - commandFailedEvent: + commandName: bulkWrite + - commandStartedEvent: + commandName: bulkWrite + - commandFailedEvent: + commandName: bulkWrite + + - - description: 'database.aggregate retries at most maxAttempts times (maxAttempts=5)' + description: 'database.aggregate retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -140,7 +267,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [aggregate] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -153,8 +280,39 @@ tests: pipeline: [ { $listLocalSessions: {} }, { $limit: 1 } ] expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + + - - description: 'database.listCollections retries at most maxAttempts times (maxAttempts=5)' + description: 'database.listCollections retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -162,7 +320,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [listCollections] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -175,8 +333,39 @@ tests: filter: {} expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + + - - description: 'database.listCollectionNames retries at most maxAttempts times (maxAttempts=5)' + description: 'database.listCollectionNames retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -184,7 +373,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [listCollections] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -197,8 +386,39 @@ tests: filter: {} expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + + - - description: 'database.runCommand retries at most maxAttempts times (maxAttempts=5)' + description: 'database.runCommand retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -206,7 +426,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [ping] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -220,8 +440,39 @@ tests: commandName: ping expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: ping + - commandFailedEvent: + commandName: ping + - commandStartedEvent: + commandName: ping + - commandFailedEvent: + commandName: ping + - commandStartedEvent: + commandName: ping + - commandFailedEvent: + commandName: ping + - commandStartedEvent: + commandName: ping + - commandFailedEvent: + commandName: ping + - commandStartedEvent: + commandName: ping + - commandFailedEvent: + commandName: ping + - commandStartedEvent: + commandName: ping + - commandFailedEvent: + commandName: ping + + - - description: 'database.createChangeStream retries at most maxAttempts times (maxAttempts=5)' + description: 'database.createChangeStream retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -229,7 +480,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [aggregate] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -243,8 +494,39 @@ tests: saveResultAsEntity: changeStream expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + + - - description: 'collection.aggregate retries at most maxAttempts times (maxAttempts=5)' + description: 'collection.aggregate retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -252,7 +534,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [aggregate] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -265,8 +547,39 @@ tests: pipeline: [] expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + + - - description: 'collection.countDocuments retries at most maxAttempts times (maxAttempts=5)' + description: 'collection.countDocuments retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -274,7 +587,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [aggregate] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -287,8 +600,39 @@ tests: filter: {} expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + + - - description: 'collection.estimatedDocumentCount retries at most maxAttempts times (maxAttempts=5)' + description: 'collection.estimatedDocumentCount retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -296,7 +640,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [count] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -307,8 +651,39 @@ tests: name: estimatedDocumentCount expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: count + - commandFailedEvent: + commandName: count + - commandStartedEvent: + commandName: count + - commandFailedEvent: + commandName: count + - commandStartedEvent: + commandName: count + - commandFailedEvent: + commandName: count + - commandStartedEvent: + commandName: count + - commandFailedEvent: + commandName: count + - commandStartedEvent: + commandName: count + - commandFailedEvent: + commandName: count + - commandStartedEvent: + commandName: count + - commandFailedEvent: + commandName: count + + - - description: 'collection.distinct retries at most maxAttempts times (maxAttempts=5)' + description: 'collection.distinct retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -316,7 +691,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [distinct] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -330,8 +705,39 @@ tests: filter: {} expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: distinct + - commandFailedEvent: + commandName: distinct + - commandStartedEvent: + commandName: distinct + - commandFailedEvent: + commandName: distinct + - commandStartedEvent: + commandName: distinct + - commandFailedEvent: + commandName: distinct + - commandStartedEvent: + commandName: distinct + - commandFailedEvent: + commandName: distinct + - commandStartedEvent: + commandName: distinct + - commandFailedEvent: + commandName: distinct + - commandStartedEvent: + commandName: distinct + - commandFailedEvent: + commandName: distinct + + - - description: 'collection.find retries at most maxAttempts times (maxAttempts=5)' + description: 'collection.find retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -339,7 +745,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [find] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -352,8 +758,39 @@ tests: filter: {} expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + + - - description: 'collection.findOne retries at most maxAttempts times (maxAttempts=5)' + description: 'collection.findOne retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -361,7 +798,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [find] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -374,8 +811,39 @@ tests: filter: {} expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + + - - description: 'collection.listIndexes retries at most maxAttempts times (maxAttempts=5)' + description: 'collection.listIndexes retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -383,7 +851,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [listIndexes] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -394,8 +862,39 @@ tests: name: listIndexes expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + + - - description: 'collection.listIndexNames retries at most maxAttempts times (maxAttempts=5)' + description: 'collection.listIndexNames retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -403,7 +902,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [listIndexes] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -414,8 +913,39 @@ tests: name: listIndexNames expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + + - - description: 'collection.createChangeStream retries at most maxAttempts times (maxAttempts=5)' + description: 'collection.createChangeStream retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -423,7 +953,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [aggregate] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -437,8 +967,39 @@ tests: saveResultAsEntity: changeStream expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + + - - description: 'collection.insertOne retries at most maxAttempts times (maxAttempts=5)' + description: 'collection.insertOne retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -446,7 +1007,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [insert] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -459,8 +1020,39 @@ tests: document: { _id: 2, x: 22 } expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + + - - description: 'collection.insertMany retries at most maxAttempts times (maxAttempts=5)' + description: 'collection.insertMany retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -468,7 +1060,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [insert] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -482,8 +1074,39 @@ tests: - { _id: 2, x: 22 } expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + + - - description: 'collection.deleteOne retries at most maxAttempts times (maxAttempts=5)' + description: 'collection.deleteOne retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -491,7 +1114,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [delete] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -504,8 +1127,39 @@ tests: filter: {} expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + + - - description: 'collection.deleteMany retries at most maxAttempts times (maxAttempts=5)' + description: 'collection.deleteMany retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -513,7 +1167,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [delete] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -526,8 +1180,39 @@ tests: filter: {} expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + + - - description: 'collection.replaceOne retries at most maxAttempts times (maxAttempts=5)' + description: 'collection.replaceOne retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -535,7 +1220,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [update] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -549,8 +1234,39 @@ tests: replacement: { x: 22 } expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + + - - description: 'collection.updateOne retries at most maxAttempts times (maxAttempts=5)' + description: 'collection.updateOne retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -558,7 +1274,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [update] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -572,8 +1288,39 @@ tests: update: { $set: { x: 22 } } expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + + - - description: 'collection.updateMany retries at most maxAttempts times (maxAttempts=5)' + description: 'collection.updateMany retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -581,7 +1328,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [update] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -595,8 +1342,39 @@ tests: update: { $set: { x: 22 } } expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + + - - description: 'collection.findOneAndDelete retries at most maxAttempts times (maxAttempts=5)' + description: 'collection.findOneAndDelete retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -604,7 +1382,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [findAndModify] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -617,8 +1395,39 @@ tests: filter: {} expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + + - - description: 'collection.findOneAndReplace retries at most maxAttempts times (maxAttempts=5)' + description: 'collection.findOneAndReplace retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -626,7 +1435,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [findAndModify] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -640,8 +1449,39 @@ tests: replacement: { x: 22 } expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + + - - description: 'collection.findOneAndUpdate retries at most maxAttempts times (maxAttempts=5)' + description: 'collection.findOneAndUpdate retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -649,7 +1489,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [findAndModify] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -663,8 +1503,39 @@ tests: update: { $set: { x: 22 } } expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + + - - description: 'collection.bulkWrite retries at most maxAttempts times (maxAttempts=5)' + description: 'collection.bulkWrite retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -672,7 +1543,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [insert] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -687,8 +1558,39 @@ tests: document: { _id: 2, x: 22 } expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + + - - description: 'collection.createIndex retries at most maxAttempts times (maxAttempts=5)' + description: 'collection.createIndex retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -696,7 +1598,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [createIndexes] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -710,8 +1612,39 @@ tests: name: "x_11" expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: createIndexes + - commandFailedEvent: + commandName: createIndexes + - commandStartedEvent: + commandName: createIndexes + - commandFailedEvent: + commandName: createIndexes + - commandStartedEvent: + commandName: createIndexes + - commandFailedEvent: + commandName: createIndexes + - commandStartedEvent: + commandName: createIndexes + - commandFailedEvent: + commandName: createIndexes + - commandStartedEvent: + commandName: createIndexes + - commandFailedEvent: + commandName: createIndexes + - commandStartedEvent: + commandName: createIndexes + - commandFailedEvent: + commandName: createIndexes + + - - description: 'collection.dropIndex retries at most maxAttempts times (maxAttempts=5)' + description: 'collection.dropIndex retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -719,7 +1652,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [dropIndexes] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -732,8 +1665,39 @@ tests: name: "x_11" expectError: true + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + + - - description: 'collection.dropIndexes retries at most maxAttempts times (maxAttempts=5)' + description: 'collection.dropIndexes retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -741,7 +1705,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [dropIndexes] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -751,3 +1715,34 @@ tests: object: *collection name: dropIndexes expectError: true + + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + diff --git a/source/client-backpressure/tests/backpressure-retry-max-attempts.yml.template b/source/client-backpressure/tests/backpressure-retry-max-attempts.yml.template index bf089211fd..555b90425d 100644 --- a/source/client-backpressure/tests/backpressure-retry-max-attempts.yml.template +++ b/source/client-backpressure/tests/backpressure-retry-max-attempts.yml.template @@ -14,6 +14,7 @@ createEntities: client: id: &client client useMultipleMongoses: false + observeEvents: [ 'commandStartedEvent', 'commandSucceededEvent', 'commandFailedEvent' ] - client: @@ -43,6 +44,10 @@ tests: {% for operation in operations %} - description: '{{operation.object}}.{{operation.operation_name}} retries at most maxAttempts=5 times' + {%- if ((operation.operation_name == 'clientBulkWrite')) %} + runOnRequirements: + minServerVersion: '8.0' + {%- endif %} operations: - name: failPoint object: testRunner @@ -50,7 +55,7 @@ tests: client: *failPointClient failPoint: configureFailPoint: failCommand - mode: { times: 6 } + mode: alwaysOn data: failCommands: [{{operation.command_name}}] errorLabels: ["RetryableError", "SystemOverloadedError"] @@ -69,4 +74,35 @@ tests: saveResultAsEntity: changeStream {%- endif %} expectError: true + + expectEvents: + - client: "client" + events: + # we expect 6 pairs of command started and succeeded events: 1 initial + # attempt and 5 retries. + - commandStartedEvent: + commandName: {{operation.command_name}} + - commandFailedEvent: + commandName: {{operation.command_name}} + - commandStartedEvent: + commandName: {{operation.command_name}} + - commandFailedEvent: + commandName: {{operation.command_name}} + - commandStartedEvent: + commandName: {{operation.command_name}} + - commandFailedEvent: + commandName: {{operation.command_name}} + - commandStartedEvent: + commandName: {{operation.command_name}} + - commandFailedEvent: + commandName: {{operation.command_name}} + - commandStartedEvent: + commandName: {{operation.command_name}} + - commandFailedEvent: + commandName: {{operation.command_name}} + - commandStartedEvent: + commandName: {{operation.command_name}} + - commandFailedEvent: + commandName: {{operation.command_name}} + {% endfor -%} From 52e2a352f30f4a7e02b4ecd7f7370b38daab6752 Mon Sep 17 00:00:00 2001 From: bailey Date: Tue, 2 Dec 2025 10:12:19 -0700 Subject: [PATCH 06/19] fix run on requirements --- .../tests/backpressure-retry-loop.json | 8 +++++--- .../client-backpressure/tests/backpressure-retry-loop.yml | 2 +- .../tests/backpressure-retry-loop.yml.template | 2 +- .../tests/backpressure-retry-max-attempts.json | 8 +++++--- .../tests/backpressure-retry-max-attempts.yml | 2 +- .../tests/backpressure-retry-max-attempts.yml.template | 2 +- 6 files changed, 14 insertions(+), 10 deletions(-) diff --git a/source/client-backpressure/tests/backpressure-retry-loop.json b/source/client-backpressure/tests/backpressure-retry-loop.json index ae944f73ad..749159d4ae 100644 --- a/source/client-backpressure/tests/backpressure-retry-loop.json +++ b/source/client-backpressure/tests/backpressure-retry-loop.json @@ -381,9 +381,11 @@ }, { "description": "client.clientBulkWrite retries using operation loop", - "runOnRequirements": { - "minServerVersion": "8.0" - }, + "runOnRequirements": [ + { + "minServerVersion": "8.0" + } + ], "operations": [ { "object": "utilCollection", diff --git a/source/client-backpressure/tests/backpressure-retry-loop.yml b/source/client-backpressure/tests/backpressure-retry-loop.yml index c612986233..c069afa6be 100644 --- a/source/client-backpressure/tests/backpressure-retry-loop.yml +++ b/source/client-backpressure/tests/backpressure-retry-loop.yml @@ -224,7 +224,7 @@ tests: - description: 'client.clientBulkWrite retries using operation loop' runOnRequirements: - minServerVersion: '8.0' + - minServerVersion: '8.0' # client bulk write added to server in 8.0 operations: - object: *utilCollection diff --git a/source/client-backpressure/tests/backpressure-retry-loop.yml.template b/source/client-backpressure/tests/backpressure-retry-loop.yml.template index cad83625d3..6101d17178 100644 --- a/source/client-backpressure/tests/backpressure-retry-loop.yml.template +++ b/source/client-backpressure/tests/backpressure-retry-loop.yml.template @@ -58,7 +58,7 @@ tests: description: '{{operation.object}}.{{operation.operation_name}} retries using operation loop' {%- if ((operation.operation_name == 'clientBulkWrite')) %} runOnRequirements: - minServerVersion: '8.0' + - minServerVersion: '8.0' # client bulk write added to server in 8.0 {%- endif %} operations: - diff --git a/source/client-backpressure/tests/backpressure-retry-max-attempts.json b/source/client-backpressure/tests/backpressure-retry-max-attempts.json index 7e9cc67d7a..5cc90248d2 100644 --- a/source/client-backpressure/tests/backpressure-retry-max-attempts.json +++ b/source/client-backpressure/tests/backpressure-retry-max-attempts.json @@ -364,9 +364,11 @@ }, { "description": "client.clientBulkWrite retries at most maxAttempts=5 times", - "runOnRequirements": { - "minServerVersion": "8.0" - }, + "runOnRequirements": [ + { + "minServerVersion": "8.0" + } + ], "operations": [ { "name": "failPoint", diff --git a/source/client-backpressure/tests/backpressure-retry-max-attempts.yml b/source/client-backpressure/tests/backpressure-retry-max-attempts.yml index 9161828e99..9bbbd74a49 100644 --- a/source/client-backpressure/tests/backpressure-retry-max-attempts.yml +++ b/source/client-backpressure/tests/backpressure-retry-max-attempts.yml @@ -203,7 +203,7 @@ tests: - description: 'client.clientBulkWrite retries at most maxAttempts=5 times' runOnRequirements: - minServerVersion: '8.0' + - minServerVersion: '8.0' # client bulk write added to server in 8.0 operations: - name: failPoint object: testRunner diff --git a/source/client-backpressure/tests/backpressure-retry-max-attempts.yml.template b/source/client-backpressure/tests/backpressure-retry-max-attempts.yml.template index 555b90425d..342556a6c3 100644 --- a/source/client-backpressure/tests/backpressure-retry-max-attempts.yml.template +++ b/source/client-backpressure/tests/backpressure-retry-max-attempts.yml.template @@ -46,7 +46,7 @@ tests: description: '{{operation.object}}.{{operation.operation_name}} retries at most maxAttempts=5 times' {%- if ((operation.operation_name == 'clientBulkWrite')) %} runOnRequirements: - minServerVersion: '8.0' + - minServerVersion: '8.0' # client bulk write added to server in 8.0 {%- endif %} operations: - name: failPoint From 391c95190bb0d1aa286ff2bd863258e43bcd0eef Mon Sep 17 00:00:00 2001 From: bailey Date: Tue, 2 Dec 2025 10:29:33 -0700 Subject: [PATCH 07/19] fix run on requirements? --- source/client-backpressure/tests/backpressure-retry-loop.json | 2 +- source/client-backpressure/tests/backpressure-retry-loop.yml | 2 +- .../tests/backpressure-retry-loop.yml.template | 2 +- .../tests/backpressure-retry-max-attempts.json | 2 +- .../tests/backpressure-retry-max-attempts.yml | 2 +- .../tests/backpressure-retry-max-attempts.yml.template | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/source/client-backpressure/tests/backpressure-retry-loop.json b/source/client-backpressure/tests/backpressure-retry-loop.json index 749159d4ae..63b3bd5e91 100644 --- a/source/client-backpressure/tests/backpressure-retry-loop.json +++ b/source/client-backpressure/tests/backpressure-retry-loop.json @@ -1,6 +1,6 @@ { "description": "tests that operations respect overload backoff retry loop", - "schemaVersion": "1.0", + "schemaVersion": "1.3", "runOnRequirements": [ { "minServerVersion": "4.4", diff --git a/source/client-backpressure/tests/backpressure-retry-loop.yml b/source/client-backpressure/tests/backpressure-retry-loop.yml index c069afa6be..8a0ea01766 100644 --- a/source/client-backpressure/tests/backpressure-retry-loop.yml +++ b/source/client-backpressure/tests/backpressure-retry-loop.yml @@ -2,7 +2,7 @@ description: tests that operations respect overload backoff retry loop -schemaVersion: '1.0' +schemaVersion: '1.3' runOnRequirements: - diff --git a/source/client-backpressure/tests/backpressure-retry-loop.yml.template b/source/client-backpressure/tests/backpressure-retry-loop.yml.template index 6101d17178..fca19f94f7 100644 --- a/source/client-backpressure/tests/backpressure-retry-loop.yml.template +++ b/source/client-backpressure/tests/backpressure-retry-loop.yml.template @@ -2,7 +2,7 @@ description: tests that operations respect overload backoff retry loop -schemaVersion: '1.0' +schemaVersion: '1.3' runOnRequirements: - diff --git a/source/client-backpressure/tests/backpressure-retry-max-attempts.json b/source/client-backpressure/tests/backpressure-retry-max-attempts.json index 5cc90248d2..206c86fe51 100644 --- a/source/client-backpressure/tests/backpressure-retry-max-attempts.json +++ b/source/client-backpressure/tests/backpressure-retry-max-attempts.json @@ -1,6 +1,6 @@ { "description": "tests that operations retry at most maxAttempts=5 times", - "schemaVersion": "1.0", + "schemaVersion": "1.3", "runOnRequirements": [ { "minServerVersion": "4.4", diff --git a/source/client-backpressure/tests/backpressure-retry-max-attempts.yml b/source/client-backpressure/tests/backpressure-retry-max-attempts.yml index 9bbbd74a49..edcac5eb87 100644 --- a/source/client-backpressure/tests/backpressure-retry-max-attempts.yml +++ b/source/client-backpressure/tests/backpressure-retry-max-attempts.yml @@ -2,7 +2,7 @@ description: tests that operations retry at most maxAttempts=5 times -schemaVersion: '1.0' +schemaVersion: '1.3' runOnRequirements: - diff --git a/source/client-backpressure/tests/backpressure-retry-max-attempts.yml.template b/source/client-backpressure/tests/backpressure-retry-max-attempts.yml.template index 342556a6c3..784fc0bb2a 100644 --- a/source/client-backpressure/tests/backpressure-retry-max-attempts.yml.template +++ b/source/client-backpressure/tests/backpressure-retry-max-attempts.yml.template @@ -2,7 +2,7 @@ description: tests that operations retry at most maxAttempts=5 times -schemaVersion: '1.0' +schemaVersion: '1.3' runOnRequirements: - From 92501c0f0e95c2c0f4d49ac9a6a8548a20a2f4f5 Mon Sep 17 00:00:00 2001 From: bailey Date: Tue, 2 Dec 2025 10:42:09 -0700 Subject: [PATCH 08/19] fix CI --- source/client-backpressure/client-backpressure.md | 4 ++++ source/logging/logging.md | 2 +- source/retryable-writes/retryable-writes.md | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/source/client-backpressure/client-backpressure.md b/source/client-backpressure/client-backpressure.md index 88dd94807e..c2bcecc545 100644 --- a/source/client-backpressure/client-backpressure.md +++ b/source/client-backpressure/client-backpressure.md @@ -390,6 +390,10 @@ attempt. Note that the second `CommandStartedEvent` and "command started" log me application from inadvertently retrying for too long (see [Backwards Compatibility](#backwards-compatibility) for details). +### Backwards Compatibility + +TODO + ## Test Plan See the [README](./tests/README.md) for tests. diff --git a/source/logging/logging.md b/source/logging/logging.md index 6f7c36c200..8971dbcde7 100644 --- a/source/logging/logging.md +++ b/source/logging/logging.md @@ -95,7 +95,7 @@ Drivers MUST support configuring where log messages should be output, including > - If the value is "stdout" (case-insensitive), log to stdout. > - If the value is "stderr" (case-insensitive), log to stderr. > - Else, if direct logging to files is supported, log to a file at the specified path. If the file already exists, it - > MUST be appended to. + > MUST be appended to. > > If the variable is not provided or is set to an invalid value (which could be invalid for any reason, e.g. the path > does not exist or is not writeable), the driver MUST log to stderr and the driver MAY attempt to warn the user about diff --git a/source/retryable-writes/retryable-writes.md b/source/retryable-writes/retryable-writes.md index fa613a2908..4a614be89a 100644 --- a/source/retryable-writes/retryable-writes.md +++ b/source/retryable-writes/retryable-writes.md @@ -50,7 +50,7 @@ across drivers. **Retryable Write Error** An error is considered retryable if it has a RetryableWriteError label in its top-level "errorLabels" field. See -[Determining Retryable Write Errors](#determining-retryable-errors) for more information. +[Determining Retryable Write Errors](#determining-retryable-write-errors) for more information. Additional terms may be defined in the [Driver Session](../sessions/driver-sessions.md) specification. From 0fdef39ac06f663ee2f38c41f71c609e856016b5 Mon Sep 17 00:00:00 2001 From: bailey Date: Wed, 3 Dec 2025 11:36:46 -0700 Subject: [PATCH 09/19] comments --- source/client-backpressure/client-backpressure.md | 7 +++++++ source/retryable-reads/retryable-reads.md | 7 ++++--- source/retryable-writes/retryable-writes.md | 7 ++++--- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/source/client-backpressure/client-backpressure.md b/source/client-backpressure/client-backpressure.md index c2bcecc545..39613c224d 100644 --- a/source/client-backpressure/client-backpressure.md +++ b/source/client-backpressure/client-backpressure.md @@ -67,6 +67,13 @@ closed"). When a new connection attempt is queued by the server for so long that the driver-side timeout expires, drivers will observe this as a network timeout error. +#### Goodput + +The throughput of positive, useful output. In the context of drivers, this refers to the number of non-error results +that the driver processes per unit of time. + +See [goodput](https://en.wikipedia.org/wiki/Goodput). + ### Requirements for Client Backpressure #### Overload retry policy diff --git a/source/retryable-reads/retryable-reads.md b/source/retryable-reads/retryable-reads.md index a7100cbe92..42046e8b9a 100644 --- a/source/retryable-reads/retryable-reads.md +++ b/source/retryable-reads/retryable-reads.md @@ -15,9 +15,10 @@ This specification will - outline how an API for retryable read operations will be implemented in drivers - define an option to enable retryable reads for an application. -The changes in this specification are related to but distinct from the retryability behaviors defined in the client -backpressure specification, which defines a retryability mechanism for all commands under certain server conditions. -Unless otherwise noted, the changes in this specification refer only to the retryability behaviors summarized above. +The changes in this specification are related to but distinct from the retryability behaviors defined in the +[Client Backpressure Specification](../client-backpressure/client-backpressure.md), which defines a retryability +mechanism for all commands under certain server conditions. Unless otherwise noted, the changes in this specification +refer only to the retryability behaviors summarized above. ## META diff --git a/source/retryable-writes/retryable-writes.md b/source/retryable-writes/retryable-writes.md index 4a614be89a..d8cf74c009 100644 --- a/source/retryable-writes/retryable-writes.md +++ b/source/retryable-writes/retryable-writes.md @@ -19,9 +19,10 @@ specification will outline how an API for retryable write operations will be imp will define an option to enable retryable writes for an application and describe how a transaction ID will be provided to write commands executed therein. -The changes in this specification are related to but distinct from the retryability behaviors defined in the client -backpressure specification, which defines a retryability mechanism for all commands under certain server conditions. -Unless otherwise noted, the changes in this specification refer only to the retryability behaviors summarized above. +The changes in this specification are related to but distinct from the retryability behaviors defined in the +[Client Backpressure Specification](../client-backpressure/client-backpressure.md), which defines a retryability +mechanism for all commands under certain server conditions. Unless otherwise noted, the changes in this specification +refer only to the retryability behaviors summarized above. ## META From 82acab8e2238497392961c1b210ab5f0d0fa86bb Mon Sep 17 00:00:00 2001 From: bailey Date: Wed, 3 Dec 2025 11:47:35 -0700 Subject: [PATCH 10/19] Fix broken unified tests --- .../tests/backpressure-retry-loop.json | 96 +++++-------- .../tests/backpressure-retry-loop.yml | 32 ----- .../backpressure-retry-loop.yml.template | 1 - .../backpressure-retry-max-attempts.json | 128 +++++++++++++----- .../tests/backpressure-retry-max-attempts.yml | 96 ++++++++----- ...ckpressure-retry-max-attempts.yml.template | 3 +- 6 files changed, 194 insertions(+), 162 deletions(-) diff --git a/source/client-backpressure/tests/backpressure-retry-loop.json b/source/client-backpressure/tests/backpressure-retry-loop.json index 63b3bd5e91..b121d9bd3d 100644 --- a/source/client-backpressure/tests/backpressure-retry-loop.json +++ b/source/client-backpressure/tests/backpressure-retry-loop.json @@ -125,8 +125,7 @@ "name": "listDatabases", "arguments": { "filter": {} - }, - "expectError": false + } } ], "expectEvents": [ @@ -224,8 +223,7 @@ }, { "object": "client", - "name": "listDatabaseNames", - "expectError": false + "name": "listDatabaseNames" } ], "expectEvents": [ @@ -327,8 +325,7 @@ "arguments": { "pipeline": [] }, - "saveResultAsEntity": "changeStream", - "expectError": false + "saveResultAsEntity": "changeStream" } ], "expectEvents": [ @@ -444,8 +441,7 @@ } } ] - }, - "expectError": false + } } ], "expectEvents": [ @@ -553,8 +549,7 @@ "$limit": 1 } ] - }, - "expectError": false + } } ], "expectEvents": [ @@ -655,8 +650,7 @@ "name": "listCollections", "arguments": { "filter": {} - }, - "expectError": false + } } ], "expectEvents": [ @@ -757,8 +751,7 @@ "name": "listCollectionNames", "arguments": { "filter": {} - }, - "expectError": false + } } ], "expectEvents": [ @@ -862,8 +855,7 @@ "ping": 1 }, "commandName": "ping" - }, - "expectError": false + } } ], "expectEvents": [ @@ -965,8 +957,7 @@ "arguments": { "pipeline": [] }, - "saveResultAsEntity": "changeStream", - "expectError": false + "saveResultAsEntity": "changeStream" } ], "expectEvents": [ @@ -1067,8 +1058,7 @@ "name": "aggregate", "arguments": { "pipeline": [] - }, - "expectError": false + } } ], "expectEvents": [ @@ -1169,8 +1159,7 @@ "name": "countDocuments", "arguments": { "filter": {} - }, - "expectError": false + } } ], "expectEvents": [ @@ -1268,8 +1257,7 @@ }, { "object": "collection", - "name": "estimatedDocumentCount", - "expectError": false + "name": "estimatedDocumentCount" } ], "expectEvents": [ @@ -1371,8 +1359,7 @@ "arguments": { "fieldName": "x", "filter": {} - }, - "expectError": false + } } ], "expectEvents": [ @@ -1473,8 +1460,7 @@ "name": "find", "arguments": { "filter": {} - }, - "expectError": false + } } ], "expectEvents": [ @@ -1575,8 +1561,7 @@ "name": "findOne", "arguments": { "filter": {} - }, - "expectError": false + } } ], "expectEvents": [ @@ -1674,8 +1659,7 @@ }, { "object": "collection", - "name": "listIndexes", - "expectError": false + "name": "listIndexes" } ], "expectEvents": [ @@ -1773,8 +1757,7 @@ }, { "object": "collection", - "name": "listIndexNames", - "expectError": false + "name": "listIndexNames" } ], "expectEvents": [ @@ -1876,8 +1859,7 @@ "arguments": { "pipeline": [] }, - "saveResultAsEntity": "changeStream", - "expectError": false + "saveResultAsEntity": "changeStream" } ], "expectEvents": [ @@ -1981,8 +1963,7 @@ "_id": 2, "x": 22 } - }, - "expectError": false + } } ], "expectEvents": [ @@ -2088,8 +2069,7 @@ "x": 22 } ] - }, - "expectError": false + } } ], "expectEvents": [ @@ -2190,8 +2170,7 @@ "name": "deleteOne", "arguments": { "filter": {} - }, - "expectError": false + } } ], "expectEvents": [ @@ -2292,8 +2271,7 @@ "name": "deleteMany", "arguments": { "filter": {} - }, - "expectError": false + } } ], "expectEvents": [ @@ -2397,8 +2375,7 @@ "replacement": { "x": 22 } - }, - "expectError": false + } } ], "expectEvents": [ @@ -2504,8 +2481,7 @@ "x": 22 } } - }, - "expectError": false + } } ], "expectEvents": [ @@ -2611,8 +2587,7 @@ "x": 22 } } - }, - "expectError": false + } } ], "expectEvents": [ @@ -2713,8 +2688,7 @@ "name": "findOneAndDelete", "arguments": { "filter": {} - }, - "expectError": false + } } ], "expectEvents": [ @@ -2818,8 +2792,7 @@ "replacement": { "x": 22 } - }, - "expectError": false + } } ], "expectEvents": [ @@ -2925,8 +2898,7 @@ "x": 22 } } - }, - "expectError": false + } } ], "expectEvents": [ @@ -3036,8 +3008,7 @@ } } ] - }, - "expectError": false + } } ], "expectEvents": [ @@ -3141,8 +3112,7 @@ "x": 11 }, "name": "x_11" - }, - "expectError": false + } } ], "expectEvents": [ @@ -3253,8 +3223,7 @@ "name": "dropIndex", "arguments": { "name": "x_11" - }, - "expectError": false + } } ], "expectEvents": [ @@ -3352,8 +3321,7 @@ }, { "object": "collection", - "name": "dropIndexes", - "expectError": false + "name": "dropIndexes" } ], "expectEvents": [ diff --git a/source/client-backpressure/tests/backpressure-retry-loop.yml b/source/client-backpressure/tests/backpressure-retry-loop.yml index 8a0ea01766..08d6072edd 100644 --- a/source/client-backpressure/tests/backpressure-retry-loop.yml +++ b/source/client-backpressure/tests/backpressure-retry-loop.yml @@ -88,7 +88,6 @@ tests: name: listDatabases arguments: filter: {} - expectError: false expectEvents: - client: "client" @@ -142,7 +141,6 @@ tests: - object: *client name: listDatabaseNames - expectError: false expectEvents: - client: "client" @@ -199,7 +197,6 @@ tests: arguments: pipeline: [] saveResultAsEntity: changeStream - expectError: false expectEvents: - client: "client" @@ -260,7 +257,6 @@ tests: - insertOne: namespace: retryable-writes-tests.coll document: { _id: 8, x: 88 } - expectError: false expectEvents: - client: "client" @@ -316,7 +312,6 @@ tests: name: aggregate arguments: pipeline: [ { $listLocalSessions: {} }, { $limit: 1 } ] - expectError: false expectEvents: - client: "client" @@ -372,7 +367,6 @@ tests: name: listCollections arguments: filter: {} - expectError: false expectEvents: - client: "client" @@ -428,7 +422,6 @@ tests: name: listCollectionNames arguments: filter: {} - expectError: false expectEvents: - client: "client" @@ -485,7 +478,6 @@ tests: arguments: command: { ping: 1 } commandName: ping - expectError: false expectEvents: - client: "client" @@ -542,7 +534,6 @@ tests: arguments: pipeline: [] saveResultAsEntity: changeStream - expectError: false expectEvents: - client: "client" @@ -598,7 +589,6 @@ tests: name: aggregate arguments: pipeline: [] - expectError: false expectEvents: - client: "client" @@ -654,7 +644,6 @@ tests: name: countDocuments arguments: filter: {} - expectError: false expectEvents: - client: "client" @@ -708,7 +697,6 @@ tests: - object: *collection name: estimatedDocumentCount - expectError: false expectEvents: - client: "client" @@ -765,7 +753,6 @@ tests: arguments: fieldName: x filter: {} - expectError: false expectEvents: - client: "client" @@ -821,7 +808,6 @@ tests: name: find arguments: filter: {} - expectError: false expectEvents: - client: "client" @@ -877,7 +863,6 @@ tests: name: findOne arguments: filter: {} - expectError: false expectEvents: - client: "client" @@ -931,7 +916,6 @@ tests: - object: *collection name: listIndexes - expectError: false expectEvents: - client: "client" @@ -985,7 +969,6 @@ tests: - object: *collection name: listIndexNames - expectError: false expectEvents: - client: "client" @@ -1042,7 +1025,6 @@ tests: arguments: pipeline: [] saveResultAsEntity: changeStream - expectError: false expectEvents: - client: "client" @@ -1098,7 +1080,6 @@ tests: name: insertOne arguments: document: { _id: 2, x: 22 } - expectError: false expectEvents: - client: "client" @@ -1155,7 +1136,6 @@ tests: arguments: documents: - { _id: 2, x: 22 } - expectError: false expectEvents: - client: "client" @@ -1211,7 +1191,6 @@ tests: name: deleteOne arguments: filter: {} - expectError: false expectEvents: - client: "client" @@ -1267,7 +1246,6 @@ tests: name: deleteMany arguments: filter: {} - expectError: false expectEvents: - client: "client" @@ -1324,7 +1302,6 @@ tests: arguments: filter: {} replacement: { x: 22 } - expectError: false expectEvents: - client: "client" @@ -1381,7 +1358,6 @@ tests: arguments: filter: {} update: { $set: { x: 22 } } - expectError: false expectEvents: - client: "client" @@ -1438,7 +1414,6 @@ tests: arguments: filter: {} update: { $set: { x: 22 } } - expectError: false expectEvents: - client: "client" @@ -1494,7 +1469,6 @@ tests: name: findOneAndDelete arguments: filter: {} - expectError: false expectEvents: - client: "client" @@ -1551,7 +1525,6 @@ tests: arguments: filter: {} replacement: { x: 22 } - expectError: false expectEvents: - client: "client" @@ -1608,7 +1581,6 @@ tests: arguments: filter: {} update: { $set: { x: 22 } } - expectError: false expectEvents: - client: "client" @@ -1666,7 +1638,6 @@ tests: requests: - insertOne: document: { _id: 2, x: 22 } - expectError: false expectEvents: - client: "client" @@ -1723,7 +1694,6 @@ tests: arguments: keys: { x: 11 } name: "x_11" - expectError: false expectEvents: - client: "client" @@ -1785,7 +1755,6 @@ tests: name: dropIndex arguments: name: "x_11" - expectError: false expectEvents: - client: "client" @@ -1839,7 +1808,6 @@ tests: - object: *collection name: dropIndexes - expectError: false expectEvents: - client: "client" diff --git a/source/client-backpressure/tests/backpressure-retry-loop.yml.template b/source/client-backpressure/tests/backpressure-retry-loop.yml.template index fca19f94f7..db3ce498e9 100644 --- a/source/client-backpressure/tests/backpressure-retry-loop.yml.template +++ b/source/client-backpressure/tests/backpressure-retry-loop.yml.template @@ -108,7 +108,6 @@ tests: {%- if operation.operation_name == "createChangeStream" %} saveResultAsEntity: changeStream {%- endif %} - expectError: false expectEvents: - client: "client" diff --git a/source/client-backpressure/tests/backpressure-retry-max-attempts.json b/source/client-backpressure/tests/backpressure-retry-max-attempts.json index 206c86fe51..d11ad9c5c5 100644 --- a/source/client-backpressure/tests/backpressure-retry-max-attempts.json +++ b/source/client-backpressure/tests/backpressure-retry-max-attempts.json @@ -91,7 +91,9 @@ "arguments": { "filter": {} }, - "expectError": true + "expectError": { + "isError": true + } } ], "expectEvents": [ @@ -189,7 +191,9 @@ { "object": "client", "name": "listDatabaseNames", - "expectError": true + "expectError": { + "isError": true + } } ], "expectEvents": [ @@ -291,7 +295,9 @@ "pipeline": [] }, "saveResultAsEntity": "changeStream", - "expectError": true + "expectError": { + "isError": true + } } ], "expectEvents": [ @@ -407,7 +413,9 @@ } ] }, - "expectError": true + "expectError": { + "isError": true + } } ], "expectEvents": [ @@ -515,7 +523,9 @@ } ] }, - "expectError": true + "expectError": { + "isError": true + } } ], "expectEvents": [ @@ -616,7 +626,9 @@ "arguments": { "filter": {} }, - "expectError": true + "expectError": { + "isError": true + } } ], "expectEvents": [ @@ -717,7 +729,9 @@ "arguments": { "filter": {} }, - "expectError": true + "expectError": { + "isError": true + } } ], "expectEvents": [ @@ -821,7 +835,9 @@ }, "commandName": "ping" }, - "expectError": true + "expectError": { + "isError": true + } } ], "expectEvents": [ @@ -923,7 +939,9 @@ "pipeline": [] }, "saveResultAsEntity": "changeStream", - "expectError": true + "expectError": { + "isError": true + } } ], "expectEvents": [ @@ -1024,7 +1042,9 @@ "arguments": { "pipeline": [] }, - "expectError": true + "expectError": { + "isError": true + } } ], "expectEvents": [ @@ -1125,7 +1145,9 @@ "arguments": { "filter": {} }, - "expectError": true + "expectError": { + "isError": true + } } ], "expectEvents": [ @@ -1223,7 +1245,9 @@ { "object": "collection", "name": "estimatedDocumentCount", - "expectError": true + "expectError": { + "isError": true + } } ], "expectEvents": [ @@ -1325,7 +1349,9 @@ "fieldName": "x", "filter": {} }, - "expectError": true + "expectError": { + "isError": true + } } ], "expectEvents": [ @@ -1426,7 +1452,9 @@ "arguments": { "filter": {} }, - "expectError": true + "expectError": { + "isError": true + } } ], "expectEvents": [ @@ -1527,7 +1555,9 @@ "arguments": { "filter": {} }, - "expectError": true + "expectError": { + "isError": true + } } ], "expectEvents": [ @@ -1625,7 +1655,9 @@ { "object": "collection", "name": "listIndexes", - "expectError": true + "expectError": { + "isError": true + } } ], "expectEvents": [ @@ -1723,7 +1755,9 @@ { "object": "collection", "name": "listIndexNames", - "expectError": true + "expectError": { + "isError": true + } } ], "expectEvents": [ @@ -1825,7 +1859,9 @@ "pipeline": [] }, "saveResultAsEntity": "changeStream", - "expectError": true + "expectError": { + "isError": true + } } ], "expectEvents": [ @@ -1929,7 +1965,9 @@ "x": 22 } }, - "expectError": true + "expectError": { + "isError": true + } } ], "expectEvents": [ @@ -2035,7 +2073,9 @@ } ] }, - "expectError": true + "expectError": { + "isError": true + } } ], "expectEvents": [ @@ -2136,7 +2176,9 @@ "arguments": { "filter": {} }, - "expectError": true + "expectError": { + "isError": true + } } ], "expectEvents": [ @@ -2237,7 +2279,9 @@ "arguments": { "filter": {} }, - "expectError": true + "expectError": { + "isError": true + } } ], "expectEvents": [ @@ -2341,7 +2385,9 @@ "x": 22 } }, - "expectError": true + "expectError": { + "isError": true + } } ], "expectEvents": [ @@ -2447,7 +2493,9 @@ } } }, - "expectError": true + "expectError": { + "isError": true + } } ], "expectEvents": [ @@ -2553,7 +2601,9 @@ } } }, - "expectError": true + "expectError": { + "isError": true + } } ], "expectEvents": [ @@ -2654,7 +2704,9 @@ "arguments": { "filter": {} }, - "expectError": true + "expectError": { + "isError": true + } } ], "expectEvents": [ @@ -2758,7 +2810,9 @@ "x": 22 } }, - "expectError": true + "expectError": { + "isError": true + } } ], "expectEvents": [ @@ -2864,7 +2918,9 @@ } } }, - "expectError": true + "expectError": { + "isError": true + } } ], "expectEvents": [ @@ -2974,7 +3030,9 @@ } ] }, - "expectError": true + "expectError": { + "isError": true + } } ], "expectEvents": [ @@ -3078,7 +3136,9 @@ }, "name": "x_11" }, - "expectError": true + "expectError": { + "isError": true + } } ], "expectEvents": [ @@ -3179,7 +3239,9 @@ "arguments": { "name": "x_11" }, - "expectError": true + "expectError": { + "isError": true + } } ], "expectEvents": [ @@ -3277,7 +3339,9 @@ { "object": "collection", "name": "dropIndexes", - "expectError": true + "expectError": { + "isError": true + } } ], "expectEvents": [ diff --git a/source/client-backpressure/tests/backpressure-retry-max-attempts.yml b/source/client-backpressure/tests/backpressure-retry-max-attempts.yml index edcac5eb87..72c9b446b7 100644 --- a/source/client-backpressure/tests/backpressure-retry-max-attempts.yml +++ b/source/client-backpressure/tests/backpressure-retry-max-attempts.yml @@ -62,7 +62,8 @@ tests: name: listDatabases arguments: filter: {} - expectError: true + expectError: + isError: true expectEvents: - client: "client" @@ -113,7 +114,8 @@ tests: - object: *client name: listDatabaseNames - expectError: true + expectError: + isError: true expectEvents: - client: "client" @@ -167,7 +169,8 @@ tests: arguments: pipeline: [] saveResultAsEntity: changeStream - expectError: true + expectError: + isError: true expectEvents: - client: "client" @@ -225,7 +228,8 @@ tests: - insertOne: namespace: retryable-writes-tests.coll document: { _id: 8, x: 88 } - expectError: true + expectError: + isError: true expectEvents: - client: "client" @@ -278,7 +282,8 @@ tests: name: aggregate arguments: pipeline: [ { $listLocalSessions: {} }, { $limit: 1 } ] - expectError: true + expectError: + isError: true expectEvents: - client: "client" @@ -331,7 +336,8 @@ tests: name: listCollections arguments: filter: {} - expectError: true + expectError: + isError: true expectEvents: - client: "client" @@ -384,7 +390,8 @@ tests: name: listCollectionNames arguments: filter: {} - expectError: true + expectError: + isError: true expectEvents: - client: "client" @@ -438,7 +445,8 @@ tests: arguments: command: { ping: 1 } commandName: ping - expectError: true + expectError: + isError: true expectEvents: - client: "client" @@ -492,7 +500,8 @@ tests: arguments: pipeline: [] saveResultAsEntity: changeStream - expectError: true + expectError: + isError: true expectEvents: - client: "client" @@ -545,7 +554,8 @@ tests: name: aggregate arguments: pipeline: [] - expectError: true + expectError: + isError: true expectEvents: - client: "client" @@ -598,7 +608,8 @@ tests: name: countDocuments arguments: filter: {} - expectError: true + expectError: + isError: true expectEvents: - client: "client" @@ -649,7 +660,8 @@ tests: - object: *collection name: estimatedDocumentCount - expectError: true + expectError: + isError: true expectEvents: - client: "client" @@ -703,7 +715,8 @@ tests: arguments: fieldName: x filter: {} - expectError: true + expectError: + isError: true expectEvents: - client: "client" @@ -756,7 +769,8 @@ tests: name: find arguments: filter: {} - expectError: true + expectError: + isError: true expectEvents: - client: "client" @@ -809,7 +823,8 @@ tests: name: findOne arguments: filter: {} - expectError: true + expectError: + isError: true expectEvents: - client: "client" @@ -860,7 +875,8 @@ tests: - object: *collection name: listIndexes - expectError: true + expectError: + isError: true expectEvents: - client: "client" @@ -911,7 +927,8 @@ tests: - object: *collection name: listIndexNames - expectError: true + expectError: + isError: true expectEvents: - client: "client" @@ -965,7 +982,8 @@ tests: arguments: pipeline: [] saveResultAsEntity: changeStream - expectError: true + expectError: + isError: true expectEvents: - client: "client" @@ -1018,7 +1036,8 @@ tests: name: insertOne arguments: document: { _id: 2, x: 22 } - expectError: true + expectError: + isError: true expectEvents: - client: "client" @@ -1072,7 +1091,8 @@ tests: arguments: documents: - { _id: 2, x: 22 } - expectError: true + expectError: + isError: true expectEvents: - client: "client" @@ -1125,7 +1145,8 @@ tests: name: deleteOne arguments: filter: {} - expectError: true + expectError: + isError: true expectEvents: - client: "client" @@ -1178,7 +1199,8 @@ tests: name: deleteMany arguments: filter: {} - expectError: true + expectError: + isError: true expectEvents: - client: "client" @@ -1232,7 +1254,8 @@ tests: arguments: filter: {} replacement: { x: 22 } - expectError: true + expectError: + isError: true expectEvents: - client: "client" @@ -1286,7 +1309,8 @@ tests: arguments: filter: {} update: { $set: { x: 22 } } - expectError: true + expectError: + isError: true expectEvents: - client: "client" @@ -1340,7 +1364,8 @@ tests: arguments: filter: {} update: { $set: { x: 22 } } - expectError: true + expectError: + isError: true expectEvents: - client: "client" @@ -1393,7 +1418,8 @@ tests: name: findOneAndDelete arguments: filter: {} - expectError: true + expectError: + isError: true expectEvents: - client: "client" @@ -1447,7 +1473,8 @@ tests: arguments: filter: {} replacement: { x: 22 } - expectError: true + expectError: + isError: true expectEvents: - client: "client" @@ -1501,7 +1528,8 @@ tests: arguments: filter: {} update: { $set: { x: 22 } } - expectError: true + expectError: + isError: true expectEvents: - client: "client" @@ -1556,7 +1584,8 @@ tests: requests: - insertOne: document: { _id: 2, x: 22 } - expectError: true + expectError: + isError: true expectEvents: - client: "client" @@ -1610,7 +1639,8 @@ tests: arguments: keys: { x: 11 } name: "x_11" - expectError: true + expectError: + isError: true expectEvents: - client: "client" @@ -1663,7 +1693,8 @@ tests: name: dropIndex arguments: name: "x_11" - expectError: true + expectError: + isError: true expectEvents: - client: "client" @@ -1714,7 +1745,8 @@ tests: - object: *collection name: dropIndexes - expectError: true + expectError: + isError: true expectEvents: - client: "client" diff --git a/source/client-backpressure/tests/backpressure-retry-max-attempts.yml.template b/source/client-backpressure/tests/backpressure-retry-max-attempts.yml.template index 784fc0bb2a..2920a4064f 100644 --- a/source/client-backpressure/tests/backpressure-retry-max-attempts.yml.template +++ b/source/client-backpressure/tests/backpressure-retry-max-attempts.yml.template @@ -73,7 +73,8 @@ tests: {%- if operation.operation_name == "createChangeStream" %} saveResultAsEntity: changeStream {%- endif %} - expectError: true + expectError: + isError: true expectEvents: - client: "client" From b3a7b6c77daa69b01667a8a13cf44cf8e11b41d0 Mon Sep 17 00:00:00 2001 From: bailey Date: Wed, 3 Dec 2025 11:58:29 -0700 Subject: [PATCH 11/19] fix UTR linting failures --- .../tests/backpressure-retry-max-attempts.json | 3 --- .../tests/backpressure-retry-max-attempts.yml | 3 --- .../tests/backpressure-retry-max-attempts.yml.template | 3 --- 3 files changed, 9 deletions(-) diff --git a/source/client-backpressure/tests/backpressure-retry-max-attempts.json b/source/client-backpressure/tests/backpressure-retry-max-attempts.json index d11ad9c5c5..a499aa490b 100644 --- a/source/client-backpressure/tests/backpressure-retry-max-attempts.json +++ b/source/client-backpressure/tests/backpressure-retry-max-attempts.json @@ -294,7 +294,6 @@ "arguments": { "pipeline": [] }, - "saveResultAsEntity": "changeStream", "expectError": { "isError": true } @@ -938,7 +937,6 @@ "arguments": { "pipeline": [] }, - "saveResultAsEntity": "changeStream", "expectError": { "isError": true } @@ -1858,7 +1856,6 @@ "arguments": { "pipeline": [] }, - "saveResultAsEntity": "changeStream", "expectError": { "isError": true } diff --git a/source/client-backpressure/tests/backpressure-retry-max-attempts.yml b/source/client-backpressure/tests/backpressure-retry-max-attempts.yml index 72c9b446b7..3bd4582757 100644 --- a/source/client-backpressure/tests/backpressure-retry-max-attempts.yml +++ b/source/client-backpressure/tests/backpressure-retry-max-attempts.yml @@ -168,7 +168,6 @@ tests: name: createChangeStream arguments: pipeline: [] - saveResultAsEntity: changeStream expectError: isError: true @@ -499,7 +498,6 @@ tests: name: createChangeStream arguments: pipeline: [] - saveResultAsEntity: changeStream expectError: isError: true @@ -981,7 +979,6 @@ tests: name: createChangeStream arguments: pipeline: [] - saveResultAsEntity: changeStream expectError: isError: true diff --git a/source/client-backpressure/tests/backpressure-retry-max-attempts.yml.template b/source/client-backpressure/tests/backpressure-retry-max-attempts.yml.template index 2920a4064f..4f2cfeee47 100644 --- a/source/client-backpressure/tests/backpressure-retry-max-attempts.yml.template +++ b/source/client-backpressure/tests/backpressure-retry-max-attempts.yml.template @@ -70,9 +70,6 @@ tests: {{arg}} {%- endfor -%} {%- endif %} - {%- if operation.operation_name == "createChangeStream" %} - saveResultAsEntity: changeStream - {%- endif %} expectError: isError: true From 60a87b8b2f0d315b822e55c66ed84d413c5fe51d Mon Sep 17 00:00:00 2001 From: bailey Date: Wed, 3 Dec 2025 15:22:28 -0700 Subject: [PATCH 12/19] remove broken deleteMany() from unified tests --- .../tests/backpressure-retry-loop.json | 384 ------------------ .../tests/backpressure-retry-loop.yml | 286 ++----------- .../backpressure-retry-loop.yml.template | 7 - 3 files changed, 31 insertions(+), 646 deletions(-) diff --git a/source/client-backpressure/tests/backpressure-retry-loop.json b/source/client-backpressure/tests/backpressure-retry-loop.json index b121d9bd3d..20bdfe3a69 100644 --- a/source/client-backpressure/tests/backpressure-retry-loop.json +++ b/source/client-backpressure/tests/backpressure-retry-loop.json @@ -85,18 +85,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -186,18 +174,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -284,18 +260,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -391,18 +355,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -502,18 +454,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -610,18 +550,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -711,18 +639,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -812,18 +728,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -916,18 +820,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -1018,18 +910,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -1119,18 +999,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -1220,18 +1088,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -1318,18 +1174,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -1420,18 +1264,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -1521,18 +1353,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -1622,18 +1442,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -1720,18 +1528,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -1818,18 +1614,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -1920,18 +1704,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -2024,18 +1796,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -2130,18 +1890,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -2231,18 +1979,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -2332,18 +2068,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -2436,18 +2160,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -2542,18 +2254,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -2648,18 +2348,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -2749,18 +2437,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -2853,18 +2529,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -2959,18 +2623,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -3069,18 +2721,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", @@ -3173,18 +2813,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "object": "utilCollection", "name": "createIndex", @@ -3284,18 +2912,6 @@ "filter": {} } }, - { - "object": "utilCollection", - "name": "deleteMany", - "arguments": { - "documents": [ - { - "_id": 1, - "x": 11 - } - ] - } - }, { "name": "failPoint", "object": "testRunner", diff --git a/source/client-backpressure/tests/backpressure-retry-loop.yml b/source/client-backpressure/tests/backpressure-retry-loop.yml index 08d6072edd..a6498568f1 100644 --- a/source/client-backpressure/tests/backpressure-retry-loop.yml +++ b/source/client-backpressure/tests/backpressure-retry-loop.yml @@ -61,14 +61,7 @@ tests: object: *utilCollection name: deleteMany arguments: - filter: {} - - - - object: *utilCollection - name: deleteMany - arguments: - documents: - - { _id: 1, x: 11 } + filter: {} - name: failPoint @@ -116,14 +109,7 @@ tests: object: *utilCollection name: deleteMany arguments: - filter: {} - - - - object: *utilCollection - name: deleteMany - arguments: - documents: - - { _id: 1, x: 11 } + filter: {} - name: failPoint @@ -169,14 +155,7 @@ tests: object: *utilCollection name: deleteMany arguments: - filter: {} - - - - object: *utilCollection - name: deleteMany - arguments: - documents: - - { _id: 1, x: 11 } + filter: {} - name: failPoint @@ -227,14 +206,7 @@ tests: object: *utilCollection name: deleteMany arguments: - filter: {} - - - - object: *utilCollection - name: deleteMany - arguments: - documents: - - { _id: 1, x: 11 } + filter: {} - name: failPoint @@ -285,14 +257,7 @@ tests: object: *utilCollection name: deleteMany arguments: - filter: {} - - - - object: *utilCollection - name: deleteMany - arguments: - documents: - - { _id: 1, x: 11 } + filter: {} - name: failPoint @@ -340,14 +305,7 @@ tests: object: *utilCollection name: deleteMany arguments: - filter: {} - - - - object: *utilCollection - name: deleteMany - arguments: - documents: - - { _id: 1, x: 11 } + filter: {} - name: failPoint @@ -395,14 +353,7 @@ tests: object: *utilCollection name: deleteMany arguments: - filter: {} - - - - object: *utilCollection - name: deleteMany - arguments: - documents: - - { _id: 1, x: 11 } + filter: {} - name: failPoint @@ -450,14 +401,7 @@ tests: object: *utilCollection name: deleteMany arguments: - filter: {} - - - - object: *utilCollection - name: deleteMany - arguments: - documents: - - { _id: 1, x: 11 } + filter: {} - name: failPoint @@ -506,14 +450,7 @@ tests: object: *utilCollection name: deleteMany arguments: - filter: {} - - - - object: *utilCollection - name: deleteMany - arguments: - documents: - - { _id: 1, x: 11 } + filter: {} - name: failPoint @@ -562,14 +499,7 @@ tests: object: *utilCollection name: deleteMany arguments: - filter: {} - - - - object: *utilCollection - name: deleteMany - arguments: - documents: - - { _id: 1, x: 11 } + filter: {} - name: failPoint @@ -617,14 +547,7 @@ tests: object: *utilCollection name: deleteMany arguments: - filter: {} - - - - object: *utilCollection - name: deleteMany - arguments: - documents: - - { _id: 1, x: 11 } + filter: {} - name: failPoint @@ -672,14 +595,7 @@ tests: object: *utilCollection name: deleteMany arguments: - filter: {} - - - - object: *utilCollection - name: deleteMany - arguments: - documents: - - { _id: 1, x: 11 } + filter: {} - name: failPoint @@ -725,14 +641,7 @@ tests: object: *utilCollection name: deleteMany arguments: - filter: {} - - - - object: *utilCollection - name: deleteMany - arguments: - documents: - - { _id: 1, x: 11 } + filter: {} - name: failPoint @@ -781,14 +690,7 @@ tests: object: *utilCollection name: deleteMany arguments: - filter: {} - - - - object: *utilCollection - name: deleteMany - arguments: - documents: - - { _id: 1, x: 11 } + filter: {} - name: failPoint @@ -836,14 +738,7 @@ tests: object: *utilCollection name: deleteMany arguments: - filter: {} - - - - object: *utilCollection - name: deleteMany - arguments: - documents: - - { _id: 1, x: 11 } + filter: {} - name: failPoint @@ -891,14 +786,7 @@ tests: object: *utilCollection name: deleteMany arguments: - filter: {} - - - - object: *utilCollection - name: deleteMany - arguments: - documents: - - { _id: 1, x: 11 } + filter: {} - name: failPoint @@ -944,14 +832,7 @@ tests: object: *utilCollection name: deleteMany arguments: - filter: {} - - - - object: *utilCollection - name: deleteMany - arguments: - documents: - - { _id: 1, x: 11 } + filter: {} - name: failPoint @@ -997,14 +878,7 @@ tests: object: *utilCollection name: deleteMany arguments: - filter: {} - - - - object: *utilCollection - name: deleteMany - arguments: - documents: - - { _id: 1, x: 11 } + filter: {} - name: failPoint @@ -1053,14 +927,7 @@ tests: object: *utilCollection name: deleteMany arguments: - filter: {} - - - - object: *utilCollection - name: deleteMany - arguments: - documents: - - { _id: 1, x: 11 } + filter: {} - name: failPoint @@ -1108,14 +975,7 @@ tests: object: *utilCollection name: deleteMany arguments: - filter: {} - - - - object: *utilCollection - name: deleteMany - arguments: - documents: - - { _id: 1, x: 11 } + filter: {} - name: failPoint @@ -1164,14 +1024,7 @@ tests: object: *utilCollection name: deleteMany arguments: - filter: {} - - - - object: *utilCollection - name: deleteMany - arguments: - documents: - - { _id: 1, x: 11 } + filter: {} - name: failPoint @@ -1219,14 +1072,7 @@ tests: object: *utilCollection name: deleteMany arguments: - filter: {} - - - - object: *utilCollection - name: deleteMany - arguments: - documents: - - { _id: 1, x: 11 } + filter: {} - name: failPoint @@ -1274,14 +1120,7 @@ tests: object: *utilCollection name: deleteMany arguments: - filter: {} - - - - object: *utilCollection - name: deleteMany - arguments: - documents: - - { _id: 1, x: 11 } + filter: {} - name: failPoint @@ -1330,14 +1169,7 @@ tests: object: *utilCollection name: deleteMany arguments: - filter: {} - - - - object: *utilCollection - name: deleteMany - arguments: - documents: - - { _id: 1, x: 11 } + filter: {} - name: failPoint @@ -1386,14 +1218,7 @@ tests: object: *utilCollection name: deleteMany arguments: - filter: {} - - - - object: *utilCollection - name: deleteMany - arguments: - documents: - - { _id: 1, x: 11 } + filter: {} - name: failPoint @@ -1442,14 +1267,7 @@ tests: object: *utilCollection name: deleteMany arguments: - filter: {} - - - - object: *utilCollection - name: deleteMany - arguments: - documents: - - { _id: 1, x: 11 } + filter: {} - name: failPoint @@ -1497,14 +1315,7 @@ tests: object: *utilCollection name: deleteMany arguments: - filter: {} - - - - object: *utilCollection - name: deleteMany - arguments: - documents: - - { _id: 1, x: 11 } + filter: {} - name: failPoint @@ -1553,14 +1364,7 @@ tests: object: *utilCollection name: deleteMany arguments: - filter: {} - - - - object: *utilCollection - name: deleteMany - arguments: - documents: - - { _id: 1, x: 11 } + filter: {} - name: failPoint @@ -1609,14 +1413,7 @@ tests: object: *utilCollection name: deleteMany arguments: - filter: {} - - - - object: *utilCollection - name: deleteMany - arguments: - documents: - - { _id: 1, x: 11 } + filter: {} - name: failPoint @@ -1666,14 +1463,7 @@ tests: object: *utilCollection name: deleteMany arguments: - filter: {} - - - - object: *utilCollection - name: deleteMany - arguments: - documents: - - { _id: 1, x: 11 } + filter: {} - name: failPoint @@ -1723,13 +1513,6 @@ tests: name: deleteMany arguments: filter: {} - - - - object: *utilCollection - name: deleteMany - arguments: - documents: - - { _id: 1, x: 11 } - object: *utilCollection name: createIndex @@ -1783,14 +1566,7 @@ tests: object: *utilCollection name: deleteMany arguments: - filter: {} - - - - object: *utilCollection - name: deleteMany - arguments: - documents: - - { _id: 1, x: 11 } + filter: {} - name: failPoint diff --git a/source/client-backpressure/tests/backpressure-retry-loop.yml.template b/source/client-backpressure/tests/backpressure-retry-loop.yml.template index db3ce498e9..53b8ba3cf6 100644 --- a/source/client-backpressure/tests/backpressure-retry-loop.yml.template +++ b/source/client-backpressure/tests/backpressure-retry-loop.yml.template @@ -67,13 +67,6 @@ tests: arguments: filter: {} - - - object: *utilCollection - name: deleteMany - arguments: - documents: - - { _id: 1, x: 11 } - {%- if operation.operation_name == "dropIndex" %} - object: *utilCollection From 399a56b72fd17a1948000bbefb548b9097cbf61c Mon Sep 17 00:00:00 2001 From: bailey Date: Wed, 10 Dec 2025 10:34:20 -0700 Subject: [PATCH 13/19] add backwards compat section --- source/client-backpressure/client-backpressure.md | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/source/client-backpressure/client-backpressure.md b/source/client-backpressure/client-backpressure.md index 39613c224d..9865546c4e 100644 --- a/source/client-backpressure/client-backpressure.md +++ b/source/client-backpressure/client-backpressure.md @@ -399,7 +399,18 @@ attempt. Note that the second `CommandStartedEvent` and "command started" log me ### Backwards Compatibility -TODO +The server's rate limiting can introduce higher error rates than previously would have been exposed to users under +periods of extreme server overload. The increased error rates is a tradeoff: given the choice between an overloaded +server (potential crash), or at minimum dramatically slower query execution time and a stable but lowered throughput +with higher error rate as the server load sheds, we have chosen the latter. + +The changes in this specification help smooth out the impact of the server's rate limiting on users by reducing the +number of errors users see during spikes or burst workloads and help prevent retry storms by spacing out retries. +However, older drivers do not have this benefit. Drivers MUST document that: + +- Users SHOULD upgrade to driver versions that officially support backpressure to avoid any impacts of server changes. +- Users who do not upgrade might see increased might need to update application error handling to handle higher error + rates of SystemOverloadedErrors. ## Test Plan From 0545e15f47540c9466689317372a22fd9b947b7d Mon Sep 17 00:00:00 2001 From: bailey Date: Thu, 11 Dec 2025 11:24:28 -0700 Subject: [PATCH 14/19] Jeff's and Jib's comments --- source/client-backpressure/client-backpressure.md | 15 +++++++-------- source/client-backpressure/tests/README.md | 2 +- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/source/client-backpressure/client-backpressure.md b/source/client-backpressure/client-backpressure.md index 9865546c4e..0f9f431f78 100644 --- a/source/client-backpressure/client-backpressure.md +++ b/source/client-backpressure/client-backpressure.md @@ -92,14 +92,15 @@ the following rules: - The value of `MAX_ATTEMPTS` is 5 and non-configurable. - This intentionally changes the behavior of CSOT which otherwise would retry an unlimited number of times within the timeout to avoid retry storms. -5. If the previous error includes the `SystemOverloadedError` label, the client MUST apply exponential backoff according - to according to the following formula: `delayMS = j * min(maxBackoff, baseBackoff * 2^i)` +5. If a retry attempt is to be attempted, a token will be consumed from the token bucket. +6. If the previous error includes the `SystemOverloadedError` label, the client MUST apply exponential backoff according + to the following formula: `delayMS = j * min(maxBackoff, baseBackoff * 2^i)` - `i` is the retry attempt (starting with 0 for the first retry). - `j` is a random jitter value between 0 and 1. - `baseBackoff` is constant 100ms. - `maxBackoff` is 10000ms. - This results in delays of 100ms, 200ms, 400ms, 800ms, and 1600ms before accounting for jitter. -6. If the previous error contained the `SystemOverloadedError` error label, the node will be added to the set of +7. If the previous error contained the `SystemOverloadedError` error label, the node will be added to the set of deprioritized servers. #### Pseudocode @@ -213,6 +214,8 @@ async function tryOperation= 2.1) ``` The sum of 5 backoffs is 3.1 seconds. There is a 1-second window to account for potential variance between the two runs. From ff5475aada6a6c10ca15337ef9cc1b241899fd81 Mon Sep 17 00:00:00 2001 From: bailey Date: Thu, 11 Dec 2025 13:19:57 -0700 Subject: [PATCH 15/19] adjust backpressure spec phrasing to make it clear retryable errors are handled separately --- .../client-backpressure.md | 74 +++++++++++++------ 1 file changed, 51 insertions(+), 23 deletions(-) diff --git a/source/client-backpressure/client-backpressure.md b/source/client-backpressure/client-backpressure.md index 0f9f431f78..1c0ba0a999 100644 --- a/source/client-backpressure/client-backpressure.md +++ b/source/client-backpressure/client-backpressure.md @@ -40,6 +40,9 @@ the connection and request rate limiters to prevent and mitigate overloading the An error is considered retryable if it includes the "RetryableError" label. This error label indicates that an operation is safely retryable regardless of the type of operation, its metadata, or any of its arguments. +Note that for the initial draft of the spec, only errors that have both the RetryableError label and the +SystemOverloadedError label are eligible for the retry backoff loop. + #### SystemOverloadedError label An error is considered overloaded if it includes the "SystemOverloadError" label. This error label indicates that the @@ -67,6 +70,9 @@ closed"). When a new connection attempt is queued by the server for so long that the driver-side timeout expires, drivers will observe this as a network timeout error. +Note that there is no guarantee that all SystemOverloaded errors are retryable or that all RetryableErrors also have the +SystemOverloaded error label. + #### Goodput The throughput of positive, useful output. In the context of drivers, this refers to the number of non-error results @@ -78,30 +84,38 @@ See [goodput](https://en.wikipedia.org/wiki/Goodput). #### Overload retry policy -This specification expands the driver's retry ability to all commands, including those not currently considered -retryable such as updateMany, create collection, getMore, and generic runCommand. The new command execution method obeys -the following rules: +This specification expands the driver's retry ability to all commands if the error indicates that is both an overload +error and that it is retryable, including those not currently considered retryable such as updateMany, create +collection, getMore, and generic runCommand. The new command execution method obeys the following rules: 1. If the command succeeds on the first attempt, drivers MUST deposit `RETRY_TOKEN_RETURN_RATE` tokens. - The value is 0.1 and non-configurable. 2. If the command succeeds on a retry attempt, drivers MUST deposit `RETRY_TOKEN_RETURN_RATE`+1 tokens. 3. If a retry attempt fails with an error that does not include `SystemOverloadedError` label, drivers MUST deposit 1 token. -4. A retry attempt will only be permitted if the error includes the `RetryableError` label, we have not reached - `MAX_ATTEMPTS`, the CSOT deadline has not expired, and a token can be acquired from the token bucket. + - A non-SystemOverloaded error indicates that the server is healthy enough to handle requests. For the purposes of + retry budget tracking, this counts as a success. +4. A retry attempt will only be permitted if the error includes the `RetryableError` label, the error has a + `SystemOverloadedError` label, we have not reached `MAX_ATTEMPTS`, the CSOT deadline has not expired, and a token + can be acquired from the token bucket. - The value of `MAX_ATTEMPTS` is 5 and non-configurable. - This intentionally changes the behavior of CSOT which otherwise would retry an unlimited number of times within the timeout to avoid retry storms. + - Note: Future work will add support for RetryableErrors to regular retryability logic (see the future work section). 5. If a retry attempt is to be attempted, a token will be consumed from the token bucket. -6. If the previous error includes the `SystemOverloadedError` label, the client MUST apply exponential backoff according - to the following formula: `delayMS = j * min(maxBackoff, baseBackoff * 2^i)` - - `i` is the retry attempt (starting with 0 for the first retry). +6. If the request is eligible for retry (as outlined in step 4), the client MUST apply exponential backoff according to + the following formula: `delayMS = j * min(maxBackoff, baseBackoff * 2^i)` + - `i` is the retry attempt number (starting with 0 for the first retry). - `j` is a random jitter value between 0 and 1. - `baseBackoff` is constant 100ms. - `maxBackoff` is 10000ms. - This results in delays of 100ms, 200ms, 400ms, 800ms, and 1600ms before accounting for jitter. -7. If the previous error contained the `SystemOverloadedError` error label, the node will be added to the set of - deprioritized servers. +7. If the request is eligible for retry (as outlined in step 4), the client MUST add the server's address to the list of + deprioritized server address for server selection. This behavior is the same as existing behavior for retryable + reads and writes. + +Note: drivers MUST share deprioritized servers between retries used for the exponential backoff loop and regular +retryable reads and writes. #### Pseudocode @@ -111,6 +125,7 @@ The following pseudocode describes the overload retry policy: BASE_BACKOFF = 0.1 MAX_BACKOFF = 10 RETRY_TOKEN_RETURN_RATE = 0.1 +MAX_ATTEMPTS = 5 def execute_command_retryable(command, ...): deprioritized_servers = [] @@ -127,19 +142,21 @@ def execute_command_retryable(command, ...): token_bucket.deposit(tokens) return res except PyMongoError as exc: - backoff = 0 + # if a retry fails with a non-System overloaded error, deposit 1 token + if attempt > 0 and not exc.has_error_label("SystemOverloadedError"): + tokens += 1 + attempt += 1 - if attempt > MAX_ATTEMPTS: + if attempt >= MAX_ATTEMPTS: raise - # Raise if the error is non retryable. - is_retryable = exc.has_error_label("RetryableError") or is_retryable_write_error() or is_retryable_read_error() - if not is_retryable: - raise error - if exc.has_error_label("SystemOverloadedError"): - jitter = random.random() # Random float between [0.0, 1.0). - backoff = jitter * min(BASE_BACKOFF * (2 ** attempt), MAX_BACKOFF) + # Raise if the error if non retryable. + if exc.has_error_label("RetryableError") and exc.has_error_label("SystemOverloadedError"): + raise + + jitter = random.random() # Random float between [0.0, 1.0). + backoff = jitter * min(BASE_BACKOFF * (2 ** attempt), MAX_BACKOFF) # If the delay exceeds the deadline, bail early before consuming a token. if _csot.get_timeout(): @@ -151,6 +168,7 @@ def execute_command_retryable(command, ...): if backoff: time.sleep(backoff) + deprioritized_servers.append(server) continue ``` @@ -161,6 +179,7 @@ that demonstrates a combined retryable reads/writes implementation with the corr from the Node driver's implementation): ```typescript +// TODO: update pseudocode with updated implementation async function tryOperation>( operation: T, { topology, timeoutContext, session, readPreference }: RetryOptions @@ -327,10 +346,10 @@ async function tryOperation Date: Thu, 11 Dec 2025 13:51:21 -0700 Subject: [PATCH 16/19] squash: jeremy's casing comments --- .../tests/backpressure-retry-loop.json | 156 ++++++------- .../tests/backpressure-retry-loop.yml | 217 +++++++++--------- .../backpressure-retry-loop.yml.template | 31 +-- 3 files changed, 187 insertions(+), 217 deletions(-) diff --git a/source/client-backpressure/tests/backpressure-retry-loop.json b/source/client-backpressure/tests/backpressure-retry-loop.json index 20bdfe3a69..4ce96b2c28 100644 --- a/source/client-backpressure/tests/backpressure-retry-loop.json +++ b/source/client-backpressure/tests/backpressure-retry-loop.json @@ -25,21 +25,21 @@ }, { "client": { - "id": "failPointClient", + "id": "internal_client", "useMultipleMongoses": false } }, { "database": { - "id": "utilDb", - "client": "failPointClient", + "id": "database", + "client": "internal_client", "databaseName": "retryable-writes-tests" } }, { "collection": { - "id": "utilCollection", - "database": "utilDb", + "id": "retryable-writes-tests", + "database": "database", "collectionName": "coll" } }, @@ -58,28 +58,12 @@ } } ], - "initialData": [ - { - "collectionName": "coll", - "databaseName": "retryable-writes-tests", - "documents": [ - { - "_id": 1, - "x": 11 - }, - { - "_id": 2, - "x": 22 - } - ] - } - ], "tests": [ { "description": "client.listDatabases retries using operation loop", "operations": [ { - "object": "utilCollection", + "object": "retryable-writes-tests", "name": "deleteMany", "arguments": { "filter": {} @@ -89,7 +73,7 @@ "name": "failPoint", "object": "testRunner", "arguments": { - "client": "failPointClient", + "client": "internal_client", "failPoint": { "configureFailPoint": "failCommand", "mode": { @@ -168,7 +152,7 @@ "description": "client.listDatabaseNames retries using operation loop", "operations": [ { - "object": "utilCollection", + "object": "retryable-writes-tests", "name": "deleteMany", "arguments": { "filter": {} @@ -178,7 +162,7 @@ "name": "failPoint", "object": "testRunner", "arguments": { - "client": "failPointClient", + "client": "internal_client", "failPoint": { "configureFailPoint": "failCommand", "mode": { @@ -254,7 +238,7 @@ "description": "client.createChangeStream retries using operation loop", "operations": [ { - "object": "utilCollection", + "object": "retryable-writes-tests", "name": "deleteMany", "arguments": { "filter": {} @@ -264,7 +248,7 @@ "name": "failPoint", "object": "testRunner", "arguments": { - "client": "failPointClient", + "client": "internal_client", "failPoint": { "configureFailPoint": "failCommand", "mode": { @@ -349,7 +333,7 @@ ], "operations": [ { - "object": "utilCollection", + "object": "retryable-writes-tests", "name": "deleteMany", "arguments": { "filter": {} @@ -359,7 +343,7 @@ "name": "failPoint", "object": "testRunner", "arguments": { - "client": "failPointClient", + "client": "internal_client", "failPoint": { "configureFailPoint": "failCommand", "mode": { @@ -448,7 +432,7 @@ "description": "database.aggregate retries using operation loop", "operations": [ { - "object": "utilCollection", + "object": "retryable-writes-tests", "name": "deleteMany", "arguments": { "filter": {} @@ -458,7 +442,7 @@ "name": "failPoint", "object": "testRunner", "arguments": { - "client": "failPointClient", + "client": "internal_client", "failPoint": { "configureFailPoint": "failCommand", "mode": { @@ -544,7 +528,7 @@ "description": "database.listCollections retries using operation loop", "operations": [ { - "object": "utilCollection", + "object": "retryable-writes-tests", "name": "deleteMany", "arguments": { "filter": {} @@ -554,7 +538,7 @@ "name": "failPoint", "object": "testRunner", "arguments": { - "client": "failPointClient", + "client": "internal_client", "failPoint": { "configureFailPoint": "failCommand", "mode": { @@ -633,7 +617,7 @@ "description": "database.listCollectionNames retries using operation loop", "operations": [ { - "object": "utilCollection", + "object": "retryable-writes-tests", "name": "deleteMany", "arguments": { "filter": {} @@ -643,7 +627,7 @@ "name": "failPoint", "object": "testRunner", "arguments": { - "client": "failPointClient", + "client": "internal_client", "failPoint": { "configureFailPoint": "failCommand", "mode": { @@ -722,7 +706,7 @@ "description": "database.runCommand retries using operation loop", "operations": [ { - "object": "utilCollection", + "object": "retryable-writes-tests", "name": "deleteMany", "arguments": { "filter": {} @@ -732,7 +716,7 @@ "name": "failPoint", "object": "testRunner", "arguments": { - "client": "failPointClient", + "client": "internal_client", "failPoint": { "configureFailPoint": "failCommand", "mode": { @@ -814,7 +798,7 @@ "description": "database.createChangeStream retries using operation loop", "operations": [ { - "object": "utilCollection", + "object": "retryable-writes-tests", "name": "deleteMany", "arguments": { "filter": {} @@ -824,7 +808,7 @@ "name": "failPoint", "object": "testRunner", "arguments": { - "client": "failPointClient", + "client": "internal_client", "failPoint": { "configureFailPoint": "failCommand", "mode": { @@ -904,7 +888,7 @@ "description": "collection.aggregate retries using operation loop", "operations": [ { - "object": "utilCollection", + "object": "retryable-writes-tests", "name": "deleteMany", "arguments": { "filter": {} @@ -914,7 +898,7 @@ "name": "failPoint", "object": "testRunner", "arguments": { - "client": "failPointClient", + "client": "internal_client", "failPoint": { "configureFailPoint": "failCommand", "mode": { @@ -993,7 +977,7 @@ "description": "collection.countDocuments retries using operation loop", "operations": [ { - "object": "utilCollection", + "object": "retryable-writes-tests", "name": "deleteMany", "arguments": { "filter": {} @@ -1003,7 +987,7 @@ "name": "failPoint", "object": "testRunner", "arguments": { - "client": "failPointClient", + "client": "internal_client", "failPoint": { "configureFailPoint": "failCommand", "mode": { @@ -1082,7 +1066,7 @@ "description": "collection.estimatedDocumentCount retries using operation loop", "operations": [ { - "object": "utilCollection", + "object": "retryable-writes-tests", "name": "deleteMany", "arguments": { "filter": {} @@ -1092,7 +1076,7 @@ "name": "failPoint", "object": "testRunner", "arguments": { - "client": "failPointClient", + "client": "internal_client", "failPoint": { "configureFailPoint": "failCommand", "mode": { @@ -1168,7 +1152,7 @@ "description": "collection.distinct retries using operation loop", "operations": [ { - "object": "utilCollection", + "object": "retryable-writes-tests", "name": "deleteMany", "arguments": { "filter": {} @@ -1178,7 +1162,7 @@ "name": "failPoint", "object": "testRunner", "arguments": { - "client": "failPointClient", + "client": "internal_client", "failPoint": { "configureFailPoint": "failCommand", "mode": { @@ -1258,7 +1242,7 @@ "description": "collection.find retries using operation loop", "operations": [ { - "object": "utilCollection", + "object": "retryable-writes-tests", "name": "deleteMany", "arguments": { "filter": {} @@ -1268,7 +1252,7 @@ "name": "failPoint", "object": "testRunner", "arguments": { - "client": "failPointClient", + "client": "internal_client", "failPoint": { "configureFailPoint": "failCommand", "mode": { @@ -1347,7 +1331,7 @@ "description": "collection.findOne retries using operation loop", "operations": [ { - "object": "utilCollection", + "object": "retryable-writes-tests", "name": "deleteMany", "arguments": { "filter": {} @@ -1357,7 +1341,7 @@ "name": "failPoint", "object": "testRunner", "arguments": { - "client": "failPointClient", + "client": "internal_client", "failPoint": { "configureFailPoint": "failCommand", "mode": { @@ -1436,7 +1420,7 @@ "description": "collection.listIndexes retries using operation loop", "operations": [ { - "object": "utilCollection", + "object": "retryable-writes-tests", "name": "deleteMany", "arguments": { "filter": {} @@ -1446,7 +1430,7 @@ "name": "failPoint", "object": "testRunner", "arguments": { - "client": "failPointClient", + "client": "internal_client", "failPoint": { "configureFailPoint": "failCommand", "mode": { @@ -1522,7 +1506,7 @@ "description": "collection.listIndexNames retries using operation loop", "operations": [ { - "object": "utilCollection", + "object": "retryable-writes-tests", "name": "deleteMany", "arguments": { "filter": {} @@ -1532,7 +1516,7 @@ "name": "failPoint", "object": "testRunner", "arguments": { - "client": "failPointClient", + "client": "internal_client", "failPoint": { "configureFailPoint": "failCommand", "mode": { @@ -1608,7 +1592,7 @@ "description": "collection.createChangeStream retries using operation loop", "operations": [ { - "object": "utilCollection", + "object": "retryable-writes-tests", "name": "deleteMany", "arguments": { "filter": {} @@ -1618,7 +1602,7 @@ "name": "failPoint", "object": "testRunner", "arguments": { - "client": "failPointClient", + "client": "internal_client", "failPoint": { "configureFailPoint": "failCommand", "mode": { @@ -1698,7 +1682,7 @@ "description": "collection.insertOne retries using operation loop", "operations": [ { - "object": "utilCollection", + "object": "retryable-writes-tests", "name": "deleteMany", "arguments": { "filter": {} @@ -1708,7 +1692,7 @@ "name": "failPoint", "object": "testRunner", "arguments": { - "client": "failPointClient", + "client": "internal_client", "failPoint": { "configureFailPoint": "failCommand", "mode": { @@ -1790,7 +1774,7 @@ "description": "collection.insertMany retries using operation loop", "operations": [ { - "object": "utilCollection", + "object": "retryable-writes-tests", "name": "deleteMany", "arguments": { "filter": {} @@ -1800,7 +1784,7 @@ "name": "failPoint", "object": "testRunner", "arguments": { - "client": "failPointClient", + "client": "internal_client", "failPoint": { "configureFailPoint": "failCommand", "mode": { @@ -1884,7 +1868,7 @@ "description": "collection.deleteOne retries using operation loop", "operations": [ { - "object": "utilCollection", + "object": "retryable-writes-tests", "name": "deleteMany", "arguments": { "filter": {} @@ -1894,7 +1878,7 @@ "name": "failPoint", "object": "testRunner", "arguments": { - "client": "failPointClient", + "client": "internal_client", "failPoint": { "configureFailPoint": "failCommand", "mode": { @@ -1973,7 +1957,7 @@ "description": "collection.deleteMany retries using operation loop", "operations": [ { - "object": "utilCollection", + "object": "retryable-writes-tests", "name": "deleteMany", "arguments": { "filter": {} @@ -1983,7 +1967,7 @@ "name": "failPoint", "object": "testRunner", "arguments": { - "client": "failPointClient", + "client": "internal_client", "failPoint": { "configureFailPoint": "failCommand", "mode": { @@ -2062,7 +2046,7 @@ "description": "collection.replaceOne retries using operation loop", "operations": [ { - "object": "utilCollection", + "object": "retryable-writes-tests", "name": "deleteMany", "arguments": { "filter": {} @@ -2072,7 +2056,7 @@ "name": "failPoint", "object": "testRunner", "arguments": { - "client": "failPointClient", + "client": "internal_client", "failPoint": { "configureFailPoint": "failCommand", "mode": { @@ -2154,7 +2138,7 @@ "description": "collection.updateOne retries using operation loop", "operations": [ { - "object": "utilCollection", + "object": "retryable-writes-tests", "name": "deleteMany", "arguments": { "filter": {} @@ -2164,7 +2148,7 @@ "name": "failPoint", "object": "testRunner", "arguments": { - "client": "failPointClient", + "client": "internal_client", "failPoint": { "configureFailPoint": "failCommand", "mode": { @@ -2248,7 +2232,7 @@ "description": "collection.updateMany retries using operation loop", "operations": [ { - "object": "utilCollection", + "object": "retryable-writes-tests", "name": "deleteMany", "arguments": { "filter": {} @@ -2258,7 +2242,7 @@ "name": "failPoint", "object": "testRunner", "arguments": { - "client": "failPointClient", + "client": "internal_client", "failPoint": { "configureFailPoint": "failCommand", "mode": { @@ -2342,7 +2326,7 @@ "description": "collection.findOneAndDelete retries using operation loop", "operations": [ { - "object": "utilCollection", + "object": "retryable-writes-tests", "name": "deleteMany", "arguments": { "filter": {} @@ -2352,7 +2336,7 @@ "name": "failPoint", "object": "testRunner", "arguments": { - "client": "failPointClient", + "client": "internal_client", "failPoint": { "configureFailPoint": "failCommand", "mode": { @@ -2431,7 +2415,7 @@ "description": "collection.findOneAndReplace retries using operation loop", "operations": [ { - "object": "utilCollection", + "object": "retryable-writes-tests", "name": "deleteMany", "arguments": { "filter": {} @@ -2441,7 +2425,7 @@ "name": "failPoint", "object": "testRunner", "arguments": { - "client": "failPointClient", + "client": "internal_client", "failPoint": { "configureFailPoint": "failCommand", "mode": { @@ -2523,7 +2507,7 @@ "description": "collection.findOneAndUpdate retries using operation loop", "operations": [ { - "object": "utilCollection", + "object": "retryable-writes-tests", "name": "deleteMany", "arguments": { "filter": {} @@ -2533,7 +2517,7 @@ "name": "failPoint", "object": "testRunner", "arguments": { - "client": "failPointClient", + "client": "internal_client", "failPoint": { "configureFailPoint": "failCommand", "mode": { @@ -2617,7 +2601,7 @@ "description": "collection.bulkWrite retries using operation loop", "operations": [ { - "object": "utilCollection", + "object": "retryable-writes-tests", "name": "deleteMany", "arguments": { "filter": {} @@ -2627,7 +2611,7 @@ "name": "failPoint", "object": "testRunner", "arguments": { - "client": "failPointClient", + "client": "internal_client", "failPoint": { "configureFailPoint": "failCommand", "mode": { @@ -2715,7 +2699,7 @@ "description": "collection.createIndex retries using operation loop", "operations": [ { - "object": "utilCollection", + "object": "retryable-writes-tests", "name": "deleteMany", "arguments": { "filter": {} @@ -2725,7 +2709,7 @@ "name": "failPoint", "object": "testRunner", "arguments": { - "client": "failPointClient", + "client": "internal_client", "failPoint": { "configureFailPoint": "failCommand", "mode": { @@ -2807,14 +2791,14 @@ "description": "collection.dropIndex retries using operation loop", "operations": [ { - "object": "utilCollection", + "object": "retryable-writes-tests", "name": "deleteMany", "arguments": { "filter": {} } }, { - "object": "utilCollection", + "object": "retryable-writes-tests", "name": "createIndex", "arguments": { "keys": { @@ -2827,7 +2811,7 @@ "name": "failPoint", "object": "testRunner", "arguments": { - "client": "failPointClient", + "client": "internal_client", "failPoint": { "configureFailPoint": "failCommand", "mode": { @@ -2906,7 +2890,7 @@ "description": "collection.dropIndexes retries using operation loop", "operations": [ { - "object": "utilCollection", + "object": "retryable-writes-tests", "name": "deleteMany", "arguments": { "filter": {} @@ -2916,7 +2900,7 @@ "name": "failPoint", "object": "testRunner", "arguments": { - "client": "failPointClient", + "client": "internal_client", "failPoint": { "configureFailPoint": "failCommand", "mode": { diff --git a/source/client-backpressure/tests/backpressure-retry-loop.yml b/source/client-backpressure/tests/backpressure-retry-loop.yml index a6498568f1..047e2b61ff 100644 --- a/source/client-backpressure/tests/backpressure-retry-loop.yml +++ b/source/client-backpressure/tests/backpressure-retry-loop.yml @@ -14,23 +14,23 @@ createEntities: client: id: &client client useMultipleMongoses: false - observeEvents: [ 'commandStartedEvent', 'commandSucceededEvent', 'commandFailedEvent' ] + observeEvents: [commandStartedEvent, commandSucceededEvent, commandFailedEvent] - client: - id: &failPointClient failPointClient + id: &internal_client internal_client useMultipleMongoses: false - database: - id: &utilDb utilDb - client: *failPointClient + id: &internal_db database + client: *internal_client databaseName: &database_name retryable-writes-tests - collection: - id: &utilCollection utilCollection - database: *utilDb + id: &internal_collection retryable-writes-tests + database: *internal_db collectionName: &collection_name coll - @@ -38,19 +38,12 @@ createEntities: id: &database database client: *client databaseName: &database_name retryable-writes-tests + - collection: id: &collection collection database: *database - collectionName: &collection_name coll - -initialData: - - - collectionName: *collection_name - databaseName: *database_name - documents: - - { _id: 1, x: 11 } - - { _id: 2, x: 22 } + collectionName: *collection_name tests: @@ -58,7 +51,7 @@ tests: description: 'client.listDatabases retries using operation loop' operations: - - object: *utilCollection + object: *internal_collection name: deleteMany arguments: filter: {} @@ -67,13 +60,13 @@ tests: - name: failPoint object: testRunner arguments: - client: *failPointClient + client: *internal_client failPoint: configureFailPoint: failCommand mode: { times: 3 } data: failCommands: [listDatabases] - errorLabels: ["RetryableError", "SystemOverloadedError"] + errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - @@ -106,7 +99,7 @@ tests: description: 'client.listDatabaseNames retries using operation loop' operations: - - object: *utilCollection + object: *internal_collection name: deleteMany arguments: filter: {} @@ -115,13 +108,13 @@ tests: - name: failPoint object: testRunner arguments: - client: *failPointClient + client: *internal_client failPoint: configureFailPoint: failCommand mode: { times: 3 } data: failCommands: [listDatabases] - errorLabels: ["RetryableError", "SystemOverloadedError"] + errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - @@ -152,7 +145,7 @@ tests: description: 'client.createChangeStream retries using operation loop' operations: - - object: *utilCollection + object: *internal_collection name: deleteMany arguments: filter: {} @@ -161,13 +154,13 @@ tests: - name: failPoint object: testRunner arguments: - client: *failPointClient + client: *internal_client failPoint: configureFailPoint: failCommand mode: { times: 3 } data: failCommands: [aggregate] - errorLabels: ["RetryableError", "SystemOverloadedError"] + errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - @@ -203,7 +196,7 @@ tests: - minServerVersion: '8.0' # client bulk write added to server in 8.0 operations: - - object: *utilCollection + object: *internal_collection name: deleteMany arguments: filter: {} @@ -212,13 +205,13 @@ tests: - name: failPoint object: testRunner arguments: - client: *failPointClient + client: *internal_client failPoint: configureFailPoint: failCommand mode: { times: 3 } data: failCommands: [bulkWrite] - errorLabels: ["RetryableError", "SystemOverloadedError"] + errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - @@ -254,7 +247,7 @@ tests: description: 'database.aggregate retries using operation loop' operations: - - object: *utilCollection + object: *internal_collection name: deleteMany arguments: filter: {} @@ -263,13 +256,13 @@ tests: - name: failPoint object: testRunner arguments: - client: *failPointClient + client: *internal_client failPoint: configureFailPoint: failCommand mode: { times: 3 } data: failCommands: [aggregate] - errorLabels: ["RetryableError", "SystemOverloadedError"] + errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - @@ -302,7 +295,7 @@ tests: description: 'database.listCollections retries using operation loop' operations: - - object: *utilCollection + object: *internal_collection name: deleteMany arguments: filter: {} @@ -311,13 +304,13 @@ tests: - name: failPoint object: testRunner arguments: - client: *failPointClient + client: *internal_client failPoint: configureFailPoint: failCommand mode: { times: 3 } data: failCommands: [listCollections] - errorLabels: ["RetryableError", "SystemOverloadedError"] + errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - @@ -350,7 +343,7 @@ tests: description: 'database.listCollectionNames retries using operation loop' operations: - - object: *utilCollection + object: *internal_collection name: deleteMany arguments: filter: {} @@ -359,13 +352,13 @@ tests: - name: failPoint object: testRunner arguments: - client: *failPointClient + client: *internal_client failPoint: configureFailPoint: failCommand mode: { times: 3 } data: failCommands: [listCollections] - errorLabels: ["RetryableError", "SystemOverloadedError"] + errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - @@ -398,7 +391,7 @@ tests: description: 'database.runCommand retries using operation loop' operations: - - object: *utilCollection + object: *internal_collection name: deleteMany arguments: filter: {} @@ -407,13 +400,13 @@ tests: - name: failPoint object: testRunner arguments: - client: *failPointClient + client: *internal_client failPoint: configureFailPoint: failCommand mode: { times: 3 } data: failCommands: [ping] - errorLabels: ["RetryableError", "SystemOverloadedError"] + errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - @@ -447,7 +440,7 @@ tests: description: 'database.createChangeStream retries using operation loop' operations: - - object: *utilCollection + object: *internal_collection name: deleteMany arguments: filter: {} @@ -456,13 +449,13 @@ tests: - name: failPoint object: testRunner arguments: - client: *failPointClient + client: *internal_client failPoint: configureFailPoint: failCommand mode: { times: 3 } data: failCommands: [aggregate] - errorLabels: ["RetryableError", "SystemOverloadedError"] + errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - @@ -496,7 +489,7 @@ tests: description: 'collection.aggregate retries using operation loop' operations: - - object: *utilCollection + object: *internal_collection name: deleteMany arguments: filter: {} @@ -505,13 +498,13 @@ tests: - name: failPoint object: testRunner arguments: - client: *failPointClient + client: *internal_client failPoint: configureFailPoint: failCommand mode: { times: 3 } data: failCommands: [aggregate] - errorLabels: ["RetryableError", "SystemOverloadedError"] + errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - @@ -544,7 +537,7 @@ tests: description: 'collection.countDocuments retries using operation loop' operations: - - object: *utilCollection + object: *internal_collection name: deleteMany arguments: filter: {} @@ -553,13 +546,13 @@ tests: - name: failPoint object: testRunner arguments: - client: *failPointClient + client: *internal_client failPoint: configureFailPoint: failCommand mode: { times: 3 } data: failCommands: [aggregate] - errorLabels: ["RetryableError", "SystemOverloadedError"] + errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - @@ -592,7 +585,7 @@ tests: description: 'collection.estimatedDocumentCount retries using operation loop' operations: - - object: *utilCollection + object: *internal_collection name: deleteMany arguments: filter: {} @@ -601,13 +594,13 @@ tests: - name: failPoint object: testRunner arguments: - client: *failPointClient + client: *internal_client failPoint: configureFailPoint: failCommand mode: { times: 3 } data: failCommands: [count] - errorLabels: ["RetryableError", "SystemOverloadedError"] + errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - @@ -638,7 +631,7 @@ tests: description: 'collection.distinct retries using operation loop' operations: - - object: *utilCollection + object: *internal_collection name: deleteMany arguments: filter: {} @@ -647,13 +640,13 @@ tests: - name: failPoint object: testRunner arguments: - client: *failPointClient + client: *internal_client failPoint: configureFailPoint: failCommand mode: { times: 3 } data: failCommands: [distinct] - errorLabels: ["RetryableError", "SystemOverloadedError"] + errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - @@ -687,7 +680,7 @@ tests: description: 'collection.find retries using operation loop' operations: - - object: *utilCollection + object: *internal_collection name: deleteMany arguments: filter: {} @@ -696,13 +689,13 @@ tests: - name: failPoint object: testRunner arguments: - client: *failPointClient + client: *internal_client failPoint: configureFailPoint: failCommand mode: { times: 3 } data: failCommands: [find] - errorLabels: ["RetryableError", "SystemOverloadedError"] + errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - @@ -735,7 +728,7 @@ tests: description: 'collection.findOne retries using operation loop' operations: - - object: *utilCollection + object: *internal_collection name: deleteMany arguments: filter: {} @@ -744,13 +737,13 @@ tests: - name: failPoint object: testRunner arguments: - client: *failPointClient + client: *internal_client failPoint: configureFailPoint: failCommand mode: { times: 3 } data: failCommands: [find] - errorLabels: ["RetryableError", "SystemOverloadedError"] + errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - @@ -783,7 +776,7 @@ tests: description: 'collection.listIndexes retries using operation loop' operations: - - object: *utilCollection + object: *internal_collection name: deleteMany arguments: filter: {} @@ -792,13 +785,13 @@ tests: - name: failPoint object: testRunner arguments: - client: *failPointClient + client: *internal_client failPoint: configureFailPoint: failCommand mode: { times: 3 } data: failCommands: [listIndexes] - errorLabels: ["RetryableError", "SystemOverloadedError"] + errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - @@ -829,7 +822,7 @@ tests: description: 'collection.listIndexNames retries using operation loop' operations: - - object: *utilCollection + object: *internal_collection name: deleteMany arguments: filter: {} @@ -838,13 +831,13 @@ tests: - name: failPoint object: testRunner arguments: - client: *failPointClient + client: *internal_client failPoint: configureFailPoint: failCommand mode: { times: 3 } data: failCommands: [listIndexes] - errorLabels: ["RetryableError", "SystemOverloadedError"] + errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - @@ -875,7 +868,7 @@ tests: description: 'collection.createChangeStream retries using operation loop' operations: - - object: *utilCollection + object: *internal_collection name: deleteMany arguments: filter: {} @@ -884,13 +877,13 @@ tests: - name: failPoint object: testRunner arguments: - client: *failPointClient + client: *internal_client failPoint: configureFailPoint: failCommand mode: { times: 3 } data: failCommands: [aggregate] - errorLabels: ["RetryableError", "SystemOverloadedError"] + errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - @@ -924,7 +917,7 @@ tests: description: 'collection.insertOne retries using operation loop' operations: - - object: *utilCollection + object: *internal_collection name: deleteMany arguments: filter: {} @@ -933,13 +926,13 @@ tests: - name: failPoint object: testRunner arguments: - client: *failPointClient + client: *internal_client failPoint: configureFailPoint: failCommand mode: { times: 3 } data: failCommands: [insert] - errorLabels: ["RetryableError", "SystemOverloadedError"] + errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - @@ -972,7 +965,7 @@ tests: description: 'collection.insertMany retries using operation loop' operations: - - object: *utilCollection + object: *internal_collection name: deleteMany arguments: filter: {} @@ -981,13 +974,13 @@ tests: - name: failPoint object: testRunner arguments: - client: *failPointClient + client: *internal_client failPoint: configureFailPoint: failCommand mode: { times: 3 } data: failCommands: [insert] - errorLabels: ["RetryableError", "SystemOverloadedError"] + errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - @@ -1021,7 +1014,7 @@ tests: description: 'collection.deleteOne retries using operation loop' operations: - - object: *utilCollection + object: *internal_collection name: deleteMany arguments: filter: {} @@ -1030,13 +1023,13 @@ tests: - name: failPoint object: testRunner arguments: - client: *failPointClient + client: *internal_client failPoint: configureFailPoint: failCommand mode: { times: 3 } data: failCommands: [delete] - errorLabels: ["RetryableError", "SystemOverloadedError"] + errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - @@ -1069,7 +1062,7 @@ tests: description: 'collection.deleteMany retries using operation loop' operations: - - object: *utilCollection + object: *internal_collection name: deleteMany arguments: filter: {} @@ -1078,13 +1071,13 @@ tests: - name: failPoint object: testRunner arguments: - client: *failPointClient + client: *internal_client failPoint: configureFailPoint: failCommand mode: { times: 3 } data: failCommands: [delete] - errorLabels: ["RetryableError", "SystemOverloadedError"] + errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - @@ -1117,7 +1110,7 @@ tests: description: 'collection.replaceOne retries using operation loop' operations: - - object: *utilCollection + object: *internal_collection name: deleteMany arguments: filter: {} @@ -1126,13 +1119,13 @@ tests: - name: failPoint object: testRunner arguments: - client: *failPointClient + client: *internal_client failPoint: configureFailPoint: failCommand mode: { times: 3 } data: failCommands: [update] - errorLabels: ["RetryableError", "SystemOverloadedError"] + errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - @@ -1166,7 +1159,7 @@ tests: description: 'collection.updateOne retries using operation loop' operations: - - object: *utilCollection + object: *internal_collection name: deleteMany arguments: filter: {} @@ -1175,13 +1168,13 @@ tests: - name: failPoint object: testRunner arguments: - client: *failPointClient + client: *internal_client failPoint: configureFailPoint: failCommand mode: { times: 3 } data: failCommands: [update] - errorLabels: ["RetryableError", "SystemOverloadedError"] + errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - @@ -1215,7 +1208,7 @@ tests: description: 'collection.updateMany retries using operation loop' operations: - - object: *utilCollection + object: *internal_collection name: deleteMany arguments: filter: {} @@ -1224,13 +1217,13 @@ tests: - name: failPoint object: testRunner arguments: - client: *failPointClient + client: *internal_client failPoint: configureFailPoint: failCommand mode: { times: 3 } data: failCommands: [update] - errorLabels: ["RetryableError", "SystemOverloadedError"] + errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - @@ -1264,7 +1257,7 @@ tests: description: 'collection.findOneAndDelete retries using operation loop' operations: - - object: *utilCollection + object: *internal_collection name: deleteMany arguments: filter: {} @@ -1273,13 +1266,13 @@ tests: - name: failPoint object: testRunner arguments: - client: *failPointClient + client: *internal_client failPoint: configureFailPoint: failCommand mode: { times: 3 } data: failCommands: [findAndModify] - errorLabels: ["RetryableError", "SystemOverloadedError"] + errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - @@ -1312,7 +1305,7 @@ tests: description: 'collection.findOneAndReplace retries using operation loop' operations: - - object: *utilCollection + object: *internal_collection name: deleteMany arguments: filter: {} @@ -1321,13 +1314,13 @@ tests: - name: failPoint object: testRunner arguments: - client: *failPointClient + client: *internal_client failPoint: configureFailPoint: failCommand mode: { times: 3 } data: failCommands: [findAndModify] - errorLabels: ["RetryableError", "SystemOverloadedError"] + errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - @@ -1361,7 +1354,7 @@ tests: description: 'collection.findOneAndUpdate retries using operation loop' operations: - - object: *utilCollection + object: *internal_collection name: deleteMany arguments: filter: {} @@ -1370,13 +1363,13 @@ tests: - name: failPoint object: testRunner arguments: - client: *failPointClient + client: *internal_client failPoint: configureFailPoint: failCommand mode: { times: 3 } data: failCommands: [findAndModify] - errorLabels: ["RetryableError", "SystemOverloadedError"] + errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - @@ -1410,7 +1403,7 @@ tests: description: 'collection.bulkWrite retries using operation loop' operations: - - object: *utilCollection + object: *internal_collection name: deleteMany arguments: filter: {} @@ -1419,13 +1412,13 @@ tests: - name: failPoint object: testRunner arguments: - client: *failPointClient + client: *internal_client failPoint: configureFailPoint: failCommand mode: { times: 3 } data: failCommands: [insert] - errorLabels: ["RetryableError", "SystemOverloadedError"] + errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - @@ -1460,7 +1453,7 @@ tests: description: 'collection.createIndex retries using operation loop' operations: - - object: *utilCollection + object: *internal_collection name: deleteMany arguments: filter: {} @@ -1469,13 +1462,13 @@ tests: - name: failPoint object: testRunner arguments: - client: *failPointClient + client: *internal_client failPoint: configureFailPoint: failCommand mode: { times: 3 } data: failCommands: [createIndexes] - errorLabels: ["RetryableError", "SystemOverloadedError"] + errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - @@ -1509,12 +1502,12 @@ tests: description: 'collection.dropIndex retries using operation loop' operations: - - object: *utilCollection + object: *internal_collection name: deleteMany arguments: filter: {} - - object: *utilCollection + object: *internal_collection name: createIndex arguments: keys: { x: 11 } @@ -1524,13 +1517,13 @@ tests: - name: failPoint object: testRunner arguments: - client: *failPointClient + client: *internal_client failPoint: configureFailPoint: failCommand mode: { times: 3 } data: failCommands: [dropIndexes] - errorLabels: ["RetryableError", "SystemOverloadedError"] + errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - @@ -1563,7 +1556,7 @@ tests: description: 'collection.dropIndexes retries using operation loop' operations: - - object: *utilCollection + object: *internal_collection name: deleteMany arguments: filter: {} @@ -1572,13 +1565,13 @@ tests: - name: failPoint object: testRunner arguments: - client: *failPointClient + client: *internal_client failPoint: configureFailPoint: failCommand mode: { times: 3 } data: failCommands: [dropIndexes] - errorLabels: ["RetryableError", "SystemOverloadedError"] + errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - diff --git a/source/client-backpressure/tests/backpressure-retry-loop.yml.template b/source/client-backpressure/tests/backpressure-retry-loop.yml.template index 53b8ba3cf6..d2f26291d3 100644 --- a/source/client-backpressure/tests/backpressure-retry-loop.yml.template +++ b/source/client-backpressure/tests/backpressure-retry-loop.yml.template @@ -14,23 +14,23 @@ createEntities: client: id: &client client useMultipleMongoses: false - observeEvents: [ 'commandStartedEvent', 'commandSucceededEvent', 'commandFailedEvent' ] + observeEvents: [commandStartedEvent, commandSucceededEvent, commandFailedEvent] - client: - id: &failPointClient failPointClient + id: &internal_client internal_client useMultipleMongoses: false - database: - id: &utilDb utilDb - client: *failPointClient + id: &internal_db database + client: *internal_client databaseName: &database_name retryable-writes-tests - collection: - id: &utilCollection utilCollection - database: *utilDb + id: &internal_collection retryable-writes-tests + database: *internal_db collectionName: &collection_name coll - @@ -38,19 +38,12 @@ createEntities: id: &database database client: *client databaseName: &database_name retryable-writes-tests + - collection: id: &collection collection database: *database - collectionName: &collection_name coll - -initialData: - - - collectionName: *collection_name - databaseName: *database_name - documents: - - { _id: 1, x: 11 } - - { _id: 2, x: 22 } + collectionName: *collection_name tests: {% for operation in operations %} @@ -62,14 +55,14 @@ tests: {%- endif %} operations: - - object: *utilCollection + object: *internal_collection name: deleteMany arguments: filter: {} {%- if operation.operation_name == "dropIndex" %} - - object: *utilCollection + object: *internal_collection name: createIndex arguments: keys: { x: 11 } @@ -80,13 +73,13 @@ tests: - name: failPoint object: testRunner arguments: - client: *failPointClient + client: *internal_client failPoint: configureFailPoint: failCommand mode: { times: 3 } data: failCommands: [{{operation.command_name}}] - errorLabels: ["RetryableError", "SystemOverloadedError"] + errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - From c1001bcd60c5ba0c10c4a859f5928d7319f8fb47 Mon Sep 17 00:00:00 2001 From: bailey Date: Thu, 11 Dec 2025 13:53:59 -0700 Subject: [PATCH 17/19] squash: other comments --- .../tests/backpressure-retry-loop.json | 9 +++------ .../tests/backpressure-retry-loop.yml | 3 --- .../tests/backpressure-retry-loop.yml.template | 3 --- 3 files changed, 3 insertions(+), 12 deletions(-) diff --git a/source/client-backpressure/tests/backpressure-retry-loop.json b/source/client-backpressure/tests/backpressure-retry-loop.json index 4ce96b2c28..79cfc4bac7 100644 --- a/source/client-backpressure/tests/backpressure-retry-loop.json +++ b/source/client-backpressure/tests/backpressure-retry-loop.json @@ -272,8 +272,7 @@ "name": "createChangeStream", "arguments": { "pipeline": [] - }, - "saveResultAsEntity": "changeStream" + } } ], "expectEvents": [ @@ -832,8 +831,7 @@ "name": "createChangeStream", "arguments": { "pipeline": [] - }, - "saveResultAsEntity": "changeStream" + } } ], "expectEvents": [ @@ -1626,8 +1624,7 @@ "name": "createChangeStream", "arguments": { "pipeline": [] - }, - "saveResultAsEntity": "changeStream" + } } ], "expectEvents": [ diff --git a/source/client-backpressure/tests/backpressure-retry-loop.yml b/source/client-backpressure/tests/backpressure-retry-loop.yml index 047e2b61ff..538ee413e2 100644 --- a/source/client-backpressure/tests/backpressure-retry-loop.yml +++ b/source/client-backpressure/tests/backpressure-retry-loop.yml @@ -168,7 +168,6 @@ tests: name: createChangeStream arguments: pipeline: [] - saveResultAsEntity: changeStream expectEvents: - client: "client" @@ -463,7 +462,6 @@ tests: name: createChangeStream arguments: pipeline: [] - saveResultAsEntity: changeStream expectEvents: - client: "client" @@ -891,7 +889,6 @@ tests: name: createChangeStream arguments: pipeline: [] - saveResultAsEntity: changeStream expectEvents: - client: "client" diff --git a/source/client-backpressure/tests/backpressure-retry-loop.yml.template b/source/client-backpressure/tests/backpressure-retry-loop.yml.template index d2f26291d3..4d80753b04 100644 --- a/source/client-backpressure/tests/backpressure-retry-loop.yml.template +++ b/source/client-backpressure/tests/backpressure-retry-loop.yml.template @@ -91,9 +91,6 @@ tests: {{arg}} {%- endfor -%} {%- endif %} - {%- if operation.operation_name == "createChangeStream" %} - saveResultAsEntity: changeStream - {%- endif %} expectEvents: - client: "client" From def5fbd0af3aee58f700a83dd87474d94bbb96a9 Mon Sep 17 00:00:00 2001 From: bailey Date: Thu, 11 Dec 2025 13:55:02 -0700 Subject: [PATCH 18/19] squash: other comments --- .../tests/backpressure-retry-loop.yml | 64 +++++++++---------- .../backpressure-retry-loop.yml.template | 2 +- 2 files changed, 33 insertions(+), 33 deletions(-) diff --git a/source/client-backpressure/tests/backpressure-retry-loop.yml b/source/client-backpressure/tests/backpressure-retry-loop.yml index 538ee413e2..a566949453 100644 --- a/source/client-backpressure/tests/backpressure-retry-loop.yml +++ b/source/client-backpressure/tests/backpressure-retry-loop.yml @@ -76,7 +76,7 @@ tests: filter: {} expectEvents: - - client: "client" + - client: *client events: - commandStartedEvent: commandName: listDatabases @@ -122,7 +122,7 @@ tests: name: listDatabaseNames expectEvents: - - client: "client" + - client: *client events: - commandStartedEvent: commandName: listDatabases @@ -170,7 +170,7 @@ tests: pipeline: [] expectEvents: - - client: "client" + - client: *client events: - commandStartedEvent: commandName: aggregate @@ -223,7 +223,7 @@ tests: document: { _id: 8, x: 88 } expectEvents: - - client: "client" + - client: *client events: - commandStartedEvent: commandName: bulkWrite @@ -271,7 +271,7 @@ tests: pipeline: [ { $listLocalSessions: {} }, { $limit: 1 } ] expectEvents: - - client: "client" + - client: *client events: - commandStartedEvent: commandName: aggregate @@ -319,7 +319,7 @@ tests: filter: {} expectEvents: - - client: "client" + - client: *client events: - commandStartedEvent: commandName: listCollections @@ -367,7 +367,7 @@ tests: filter: {} expectEvents: - - client: "client" + - client: *client events: - commandStartedEvent: commandName: listCollections @@ -416,7 +416,7 @@ tests: commandName: ping expectEvents: - - client: "client" + - client: *client events: - commandStartedEvent: commandName: ping @@ -464,7 +464,7 @@ tests: pipeline: [] expectEvents: - - client: "client" + - client: *client events: - commandStartedEvent: commandName: aggregate @@ -512,7 +512,7 @@ tests: pipeline: [] expectEvents: - - client: "client" + - client: *client events: - commandStartedEvent: commandName: aggregate @@ -560,7 +560,7 @@ tests: filter: {} expectEvents: - - client: "client" + - client: *client events: - commandStartedEvent: commandName: aggregate @@ -606,7 +606,7 @@ tests: name: estimatedDocumentCount expectEvents: - - client: "client" + - client: *client events: - commandStartedEvent: commandName: count @@ -655,7 +655,7 @@ tests: filter: {} expectEvents: - - client: "client" + - client: *client events: - commandStartedEvent: commandName: distinct @@ -703,7 +703,7 @@ tests: filter: {} expectEvents: - - client: "client" + - client: *client events: - commandStartedEvent: commandName: find @@ -751,7 +751,7 @@ tests: filter: {} expectEvents: - - client: "client" + - client: *client events: - commandStartedEvent: commandName: find @@ -797,7 +797,7 @@ tests: name: listIndexes expectEvents: - - client: "client" + - client: *client events: - commandStartedEvent: commandName: listIndexes @@ -843,7 +843,7 @@ tests: name: listIndexNames expectEvents: - - client: "client" + - client: *client events: - commandStartedEvent: commandName: listIndexes @@ -891,7 +891,7 @@ tests: pipeline: [] expectEvents: - - client: "client" + - client: *client events: - commandStartedEvent: commandName: aggregate @@ -939,7 +939,7 @@ tests: document: { _id: 2, x: 22 } expectEvents: - - client: "client" + - client: *client events: - commandStartedEvent: commandName: insert @@ -988,7 +988,7 @@ tests: - { _id: 2, x: 22 } expectEvents: - - client: "client" + - client: *client events: - commandStartedEvent: commandName: insert @@ -1036,7 +1036,7 @@ tests: filter: {} expectEvents: - - client: "client" + - client: *client events: - commandStartedEvent: commandName: delete @@ -1084,7 +1084,7 @@ tests: filter: {} expectEvents: - - client: "client" + - client: *client events: - commandStartedEvent: commandName: delete @@ -1133,7 +1133,7 @@ tests: replacement: { x: 22 } expectEvents: - - client: "client" + - client: *client events: - commandStartedEvent: commandName: update @@ -1182,7 +1182,7 @@ tests: update: { $set: { x: 22 } } expectEvents: - - client: "client" + - client: *client events: - commandStartedEvent: commandName: update @@ -1231,7 +1231,7 @@ tests: update: { $set: { x: 22 } } expectEvents: - - client: "client" + - client: *client events: - commandStartedEvent: commandName: update @@ -1279,7 +1279,7 @@ tests: filter: {} expectEvents: - - client: "client" + - client: *client events: - commandStartedEvent: commandName: findAndModify @@ -1328,7 +1328,7 @@ tests: replacement: { x: 22 } expectEvents: - - client: "client" + - client: *client events: - commandStartedEvent: commandName: findAndModify @@ -1377,7 +1377,7 @@ tests: update: { $set: { x: 22 } } expectEvents: - - client: "client" + - client: *client events: - commandStartedEvent: commandName: findAndModify @@ -1427,7 +1427,7 @@ tests: document: { _id: 2, x: 22 } expectEvents: - - client: "client" + - client: *client events: - commandStartedEvent: commandName: insert @@ -1476,7 +1476,7 @@ tests: name: "x_11" expectEvents: - - client: "client" + - client: *client events: - commandStartedEvent: commandName: createIndexes @@ -1530,7 +1530,7 @@ tests: name: "x_11" expectEvents: - - client: "client" + - client: *client events: - commandStartedEvent: commandName: dropIndexes @@ -1576,7 +1576,7 @@ tests: name: dropIndexes expectEvents: - - client: "client" + - client: *client events: - commandStartedEvent: commandName: dropIndexes diff --git a/source/client-backpressure/tests/backpressure-retry-loop.yml.template b/source/client-backpressure/tests/backpressure-retry-loop.yml.template index 4d80753b04..ac47783e53 100644 --- a/source/client-backpressure/tests/backpressure-retry-loop.yml.template +++ b/source/client-backpressure/tests/backpressure-retry-loop.yml.template @@ -93,7 +93,7 @@ tests: {%- endif %} expectEvents: - - client: "client" + - client: *client events: - commandStartedEvent: commandName: {{operation.command_name}} From 08da5c4fa872180a56c4fa21fa13d869fffd89d8 Mon Sep 17 00:00:00 2001 From: bailey Date: Thu, 11 Dec 2025 14:13:26 -0700 Subject: [PATCH 19/19] last round comments --- source/client-backpressure/tests/README.md | 12 +- .../backpressure-retry-max-attempts.json | 162 +++++--- .../tests/backpressure-retry-max-attempts.yml | 388 ++++++++++-------- ...ckpressure-retry-max-attempts.yml.template | 16 +- 4 files changed, 338 insertions(+), 240 deletions(-) diff --git a/source/client-backpressure/tests/README.md b/source/client-backpressure/tests/README.md index 41ad017c6c..b4b03085e6 100644 --- a/source/client-backpressure/tests/README.md +++ b/source/client-backpressure/tests/README.md @@ -16,8 +16,8 @@ be manually implemented by each driver. Drivers should test that retries do not occur immediately when a SystemOverloadedError is encountered. -1. let `client` be a `MongoClient` -2. let `collection` be a collection +1. Let `client` be a `MongoClient` +2. Let `collection` be a collection 3. Now, run transactions without backoff: 1. Configure the random number generator used for jitter to always return `0` -- this effectively disables backoff. @@ -28,14 +28,14 @@ Drivers should test that retries do not occur immediately when a SystemOverloade configureFailPoint: 'failCommand', mode: 'alwaysOn', data: { - failCommands: ['insert'], - errorCode: 2, - errorLabels: ['SystemOverloadedError', 'RetryableError'] + failCommands: ['insert'], + errorCode: 2, + errorLabels: ['SystemOverloadedError', 'RetryableError'] } } ``` - 3. Execute the following command. Expect that the command errors. Measure the duration of the command execution. + 3. Execute the document `{ a: 1 }`. Expect that the command errors. Measure the duration of the command execution. ```javascript const start = performance.now(); diff --git a/source/client-backpressure/tests/backpressure-retry-max-attempts.json b/source/client-backpressure/tests/backpressure-retry-max-attempts.json index a499aa490b..0cd52e9a53 100644 --- a/source/client-backpressure/tests/backpressure-retry-max-attempts.json +++ b/source/client-backpressure/tests/backpressure-retry-max-attempts.json @@ -25,7 +25,7 @@ }, { "client": { - "id": "failPointClient", + "id": "fail_point_client", "useMultipleMongoses": false } }, @@ -68,7 +68,7 @@ "name": "failPoint", "object": "testRunner", "arguments": { - "client": "failPointClient", + "client": "fail_point_client", "failPoint": { "configureFailPoint": "failCommand", "mode": "alwaysOn", @@ -92,7 +92,8 @@ "filter": {} }, "expectError": { - "isError": true + "isError": true, + "isClientError": false } } ], @@ -171,7 +172,7 @@ "name": "failPoint", "object": "testRunner", "arguments": { - "client": "failPointClient", + "client": "fail_point_client", "failPoint": { "configureFailPoint": "failCommand", "mode": "alwaysOn", @@ -192,7 +193,8 @@ "object": "client", "name": "listDatabaseNames", "expectError": { - "isError": true + "isError": true, + "isClientError": false } } ], @@ -271,7 +273,7 @@ "name": "failPoint", "object": "testRunner", "arguments": { - "client": "failPointClient", + "client": "fail_point_client", "failPoint": { "configureFailPoint": "failCommand", "mode": "alwaysOn", @@ -295,7 +297,8 @@ "pipeline": [] }, "expectError": { - "isError": true + "isError": true, + "isClientError": false } } ], @@ -379,7 +382,7 @@ "name": "failPoint", "object": "testRunner", "arguments": { - "client": "failPointClient", + "client": "fail_point_client", "failPoint": { "configureFailPoint": "failCommand", "mode": "alwaysOn", @@ -413,7 +416,8 @@ ] }, "expectError": { - "isError": true + "isError": true, + "isClientError": false } } ], @@ -492,7 +496,7 @@ "name": "failPoint", "object": "testRunner", "arguments": { - "client": "failPointClient", + "client": "fail_point_client", "failPoint": { "configureFailPoint": "failCommand", "mode": "alwaysOn", @@ -523,7 +527,8 @@ ] }, "expectError": { - "isError": true + "isError": true, + "isClientError": false } } ], @@ -602,7 +607,7 @@ "name": "failPoint", "object": "testRunner", "arguments": { - "client": "failPointClient", + "client": "fail_point_client", "failPoint": { "configureFailPoint": "failCommand", "mode": "alwaysOn", @@ -626,7 +631,8 @@ "filter": {} }, "expectError": { - "isError": true + "isError": true, + "isClientError": false } } ], @@ -705,7 +711,7 @@ "name": "failPoint", "object": "testRunner", "arguments": { - "client": "failPointClient", + "client": "fail_point_client", "failPoint": { "configureFailPoint": "failCommand", "mode": "alwaysOn", @@ -729,7 +735,8 @@ "filter": {} }, "expectError": { - "isError": true + "isError": true, + "isClientError": false } } ], @@ -808,7 +815,7 @@ "name": "failPoint", "object": "testRunner", "arguments": { - "client": "failPointClient", + "client": "fail_point_client", "failPoint": { "configureFailPoint": "failCommand", "mode": "alwaysOn", @@ -835,7 +842,8 @@ "commandName": "ping" }, "expectError": { - "isError": true + "isError": true, + "isClientError": false } } ], @@ -914,7 +922,7 @@ "name": "failPoint", "object": "testRunner", "arguments": { - "client": "failPointClient", + "client": "fail_point_client", "failPoint": { "configureFailPoint": "failCommand", "mode": "alwaysOn", @@ -938,7 +946,8 @@ "pipeline": [] }, "expectError": { - "isError": true + "isError": true, + "isClientError": false } } ], @@ -1017,7 +1026,7 @@ "name": "failPoint", "object": "testRunner", "arguments": { - "client": "failPointClient", + "client": "fail_point_client", "failPoint": { "configureFailPoint": "failCommand", "mode": "alwaysOn", @@ -1041,7 +1050,8 @@ "pipeline": [] }, "expectError": { - "isError": true + "isError": true, + "isClientError": false } } ], @@ -1120,7 +1130,7 @@ "name": "failPoint", "object": "testRunner", "arguments": { - "client": "failPointClient", + "client": "fail_point_client", "failPoint": { "configureFailPoint": "failCommand", "mode": "alwaysOn", @@ -1144,7 +1154,8 @@ "filter": {} }, "expectError": { - "isError": true + "isError": true, + "isClientError": false } } ], @@ -1223,7 +1234,7 @@ "name": "failPoint", "object": "testRunner", "arguments": { - "client": "failPointClient", + "client": "fail_point_client", "failPoint": { "configureFailPoint": "failCommand", "mode": "alwaysOn", @@ -1244,7 +1255,8 @@ "object": "collection", "name": "estimatedDocumentCount", "expectError": { - "isError": true + "isError": true, + "isClientError": false } } ], @@ -1323,7 +1335,7 @@ "name": "failPoint", "object": "testRunner", "arguments": { - "client": "failPointClient", + "client": "fail_point_client", "failPoint": { "configureFailPoint": "failCommand", "mode": "alwaysOn", @@ -1348,7 +1360,8 @@ "filter": {} }, "expectError": { - "isError": true + "isError": true, + "isClientError": false } } ], @@ -1427,7 +1440,7 @@ "name": "failPoint", "object": "testRunner", "arguments": { - "client": "failPointClient", + "client": "fail_point_client", "failPoint": { "configureFailPoint": "failCommand", "mode": "alwaysOn", @@ -1451,7 +1464,8 @@ "filter": {} }, "expectError": { - "isError": true + "isError": true, + "isClientError": false } } ], @@ -1530,7 +1544,7 @@ "name": "failPoint", "object": "testRunner", "arguments": { - "client": "failPointClient", + "client": "fail_point_client", "failPoint": { "configureFailPoint": "failCommand", "mode": "alwaysOn", @@ -1554,7 +1568,8 @@ "filter": {} }, "expectError": { - "isError": true + "isError": true, + "isClientError": false } } ], @@ -1633,7 +1648,7 @@ "name": "failPoint", "object": "testRunner", "arguments": { - "client": "failPointClient", + "client": "fail_point_client", "failPoint": { "configureFailPoint": "failCommand", "mode": "alwaysOn", @@ -1654,7 +1669,8 @@ "object": "collection", "name": "listIndexes", "expectError": { - "isError": true + "isError": true, + "isClientError": false } } ], @@ -1733,7 +1749,7 @@ "name": "failPoint", "object": "testRunner", "arguments": { - "client": "failPointClient", + "client": "fail_point_client", "failPoint": { "configureFailPoint": "failCommand", "mode": "alwaysOn", @@ -1754,7 +1770,8 @@ "object": "collection", "name": "listIndexNames", "expectError": { - "isError": true + "isError": true, + "isClientError": false } } ], @@ -1833,7 +1850,7 @@ "name": "failPoint", "object": "testRunner", "arguments": { - "client": "failPointClient", + "client": "fail_point_client", "failPoint": { "configureFailPoint": "failCommand", "mode": "alwaysOn", @@ -1857,7 +1874,8 @@ "pipeline": [] }, "expectError": { - "isError": true + "isError": true, + "isClientError": false } } ], @@ -1936,7 +1954,7 @@ "name": "failPoint", "object": "testRunner", "arguments": { - "client": "failPointClient", + "client": "fail_point_client", "failPoint": { "configureFailPoint": "failCommand", "mode": "alwaysOn", @@ -1963,7 +1981,8 @@ } }, "expectError": { - "isError": true + "isError": true, + "isClientError": false } } ], @@ -2042,7 +2061,7 @@ "name": "failPoint", "object": "testRunner", "arguments": { - "client": "failPointClient", + "client": "fail_point_client", "failPoint": { "configureFailPoint": "failCommand", "mode": "alwaysOn", @@ -2071,7 +2090,8 @@ ] }, "expectError": { - "isError": true + "isError": true, + "isClientError": false } } ], @@ -2150,7 +2170,7 @@ "name": "failPoint", "object": "testRunner", "arguments": { - "client": "failPointClient", + "client": "fail_point_client", "failPoint": { "configureFailPoint": "failCommand", "mode": "alwaysOn", @@ -2174,7 +2194,8 @@ "filter": {} }, "expectError": { - "isError": true + "isError": true, + "isClientError": false } } ], @@ -2253,7 +2274,7 @@ "name": "failPoint", "object": "testRunner", "arguments": { - "client": "failPointClient", + "client": "fail_point_client", "failPoint": { "configureFailPoint": "failCommand", "mode": "alwaysOn", @@ -2277,7 +2298,8 @@ "filter": {} }, "expectError": { - "isError": true + "isError": true, + "isClientError": false } } ], @@ -2356,7 +2378,7 @@ "name": "failPoint", "object": "testRunner", "arguments": { - "client": "failPointClient", + "client": "fail_point_client", "failPoint": { "configureFailPoint": "failCommand", "mode": "alwaysOn", @@ -2383,7 +2405,8 @@ } }, "expectError": { - "isError": true + "isError": true, + "isClientError": false } } ], @@ -2462,7 +2485,7 @@ "name": "failPoint", "object": "testRunner", "arguments": { - "client": "failPointClient", + "client": "fail_point_client", "failPoint": { "configureFailPoint": "failCommand", "mode": "alwaysOn", @@ -2491,7 +2514,8 @@ } }, "expectError": { - "isError": true + "isError": true, + "isClientError": false } } ], @@ -2570,7 +2594,7 @@ "name": "failPoint", "object": "testRunner", "arguments": { - "client": "failPointClient", + "client": "fail_point_client", "failPoint": { "configureFailPoint": "failCommand", "mode": "alwaysOn", @@ -2599,7 +2623,8 @@ } }, "expectError": { - "isError": true + "isError": true, + "isClientError": false } } ], @@ -2678,7 +2703,7 @@ "name": "failPoint", "object": "testRunner", "arguments": { - "client": "failPointClient", + "client": "fail_point_client", "failPoint": { "configureFailPoint": "failCommand", "mode": "alwaysOn", @@ -2702,7 +2727,8 @@ "filter": {} }, "expectError": { - "isError": true + "isError": true, + "isClientError": false } } ], @@ -2781,7 +2807,7 @@ "name": "failPoint", "object": "testRunner", "arguments": { - "client": "failPointClient", + "client": "fail_point_client", "failPoint": { "configureFailPoint": "failCommand", "mode": "alwaysOn", @@ -2808,7 +2834,8 @@ } }, "expectError": { - "isError": true + "isError": true, + "isClientError": false } } ], @@ -2887,7 +2914,7 @@ "name": "failPoint", "object": "testRunner", "arguments": { - "client": "failPointClient", + "client": "fail_point_client", "failPoint": { "configureFailPoint": "failCommand", "mode": "alwaysOn", @@ -2916,7 +2943,8 @@ } }, "expectError": { - "isError": true + "isError": true, + "isClientError": false } } ], @@ -2995,7 +3023,7 @@ "name": "failPoint", "object": "testRunner", "arguments": { - "client": "failPointClient", + "client": "fail_point_client", "failPoint": { "configureFailPoint": "failCommand", "mode": "alwaysOn", @@ -3028,7 +3056,8 @@ ] }, "expectError": { - "isError": true + "isError": true, + "isClientError": false } } ], @@ -3107,7 +3136,7 @@ "name": "failPoint", "object": "testRunner", "arguments": { - "client": "failPointClient", + "client": "fail_point_client", "failPoint": { "configureFailPoint": "failCommand", "mode": "alwaysOn", @@ -3134,7 +3163,8 @@ "name": "x_11" }, "expectError": { - "isError": true + "isError": true, + "isClientError": false } } ], @@ -3213,7 +3243,7 @@ "name": "failPoint", "object": "testRunner", "arguments": { - "client": "failPointClient", + "client": "fail_point_client", "failPoint": { "configureFailPoint": "failCommand", "mode": "alwaysOn", @@ -3237,7 +3267,8 @@ "name": "x_11" }, "expectError": { - "isError": true + "isError": true, + "isClientError": false } } ], @@ -3316,7 +3347,7 @@ "name": "failPoint", "object": "testRunner", "arguments": { - "client": "failPointClient", + "client": "fail_point_client", "failPoint": { "configureFailPoint": "failCommand", "mode": "alwaysOn", @@ -3337,7 +3368,8 @@ "object": "collection", "name": "dropIndexes", "expectError": { - "isError": true + "isError": true, + "isClientError": false } } ], diff --git a/source/client-backpressure/tests/backpressure-retry-max-attempts.yml b/source/client-backpressure/tests/backpressure-retry-max-attempts.yml index 3bd4582757..05da92d671 100644 --- a/source/client-backpressure/tests/backpressure-retry-max-attempts.yml +++ b/source/client-backpressure/tests/backpressure-retry-max-attempts.yml @@ -14,11 +14,11 @@ createEntities: client: id: &client client useMultipleMongoses: false - observeEvents: [ 'commandStartedEvent', 'commandSucceededEvent', 'commandFailedEvent' ] + observeEvents: [commandStartedEvent, commandSucceededEvent, commandFailedEvent] - client: - id: &failPointClient failPointClient + id: &fail_point_client fail_point_client useMultipleMongoses: false - @@ -44,17 +44,18 @@ tests: - description: 'client.listDatabases retries at most maxAttempts=5 times' + operations: - name: failPoint object: testRunner arguments: - client: *failPointClient + client: *fail_point_client failPoint: configureFailPoint: failCommand mode: alwaysOn data: failCommands: [listDatabases] - errorLabels: ["RetryableError", "SystemOverloadedError"] + errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - @@ -64,12 +65,13 @@ tests: filter: {} expectError: isError: true + isClientError: false expectEvents: - - client: "client" + - client: *client events: - # we expect 6 pairs of command started and succeeded events: 1 initial - # attempt and 5 retries. + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. - commandStartedEvent: commandName: listDatabases - commandFailedEvent: @@ -98,17 +100,18 @@ tests: - description: 'client.listDatabaseNames retries at most maxAttempts=5 times' + operations: - name: failPoint object: testRunner arguments: - client: *failPointClient + client: *fail_point_client failPoint: configureFailPoint: failCommand mode: alwaysOn data: failCommands: [listDatabases] - errorLabels: ["RetryableError", "SystemOverloadedError"] + errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - @@ -116,12 +119,13 @@ tests: name: listDatabaseNames expectError: isError: true + isClientError: false expectEvents: - - client: "client" + - client: *client events: - # we expect 6 pairs of command started and succeeded events: 1 initial - # attempt and 5 retries. + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. - commandStartedEvent: commandName: listDatabases - commandFailedEvent: @@ -150,17 +154,18 @@ tests: - description: 'client.createChangeStream retries at most maxAttempts=5 times' + operations: - name: failPoint object: testRunner arguments: - client: *failPointClient + client: *fail_point_client failPoint: configureFailPoint: failCommand mode: alwaysOn data: failCommands: [aggregate] - errorLabels: ["RetryableError", "SystemOverloadedError"] + errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - @@ -170,12 +175,13 @@ tests: pipeline: [] expectError: isError: true + isClientError: false expectEvents: - - client: "client" + - client: *client events: - # we expect 6 pairs of command started and succeeded events: 1 initial - # attempt and 5 retries. + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. - commandStartedEvent: commandName: aggregate - commandFailedEvent: @@ -206,17 +212,18 @@ tests: description: 'client.clientBulkWrite retries at most maxAttempts=5 times' runOnRequirements: - minServerVersion: '8.0' # client bulk write added to server in 8.0 + operations: - name: failPoint object: testRunner arguments: - client: *failPointClient + client: *fail_point_client failPoint: configureFailPoint: failCommand mode: alwaysOn data: failCommands: [bulkWrite] - errorLabels: ["RetryableError", "SystemOverloadedError"] + errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - @@ -229,12 +236,13 @@ tests: document: { _id: 8, x: 88 } expectError: isError: true + isClientError: false expectEvents: - - client: "client" + - client: *client events: - # we expect 6 pairs of command started and succeeded events: 1 initial - # attempt and 5 retries. + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. - commandStartedEvent: commandName: bulkWrite - commandFailedEvent: @@ -263,17 +271,18 @@ tests: - description: 'database.aggregate retries at most maxAttempts=5 times' + operations: - name: failPoint object: testRunner arguments: - client: *failPointClient + client: *fail_point_client failPoint: configureFailPoint: failCommand mode: alwaysOn data: failCommands: [aggregate] - errorLabels: ["RetryableError", "SystemOverloadedError"] + errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - @@ -283,12 +292,13 @@ tests: pipeline: [ { $listLocalSessions: {} }, { $limit: 1 } ] expectError: isError: true + isClientError: false expectEvents: - - client: "client" + - client: *client events: - # we expect 6 pairs of command started and succeeded events: 1 initial - # attempt and 5 retries. + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. - commandStartedEvent: commandName: aggregate - commandFailedEvent: @@ -317,17 +327,18 @@ tests: - description: 'database.listCollections retries at most maxAttempts=5 times' + operations: - name: failPoint object: testRunner arguments: - client: *failPointClient + client: *fail_point_client failPoint: configureFailPoint: failCommand mode: alwaysOn data: failCommands: [listCollections] - errorLabels: ["RetryableError", "SystemOverloadedError"] + errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - @@ -337,12 +348,13 @@ tests: filter: {} expectError: isError: true + isClientError: false expectEvents: - - client: "client" + - client: *client events: - # we expect 6 pairs of command started and succeeded events: 1 initial - # attempt and 5 retries. + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. - commandStartedEvent: commandName: listCollections - commandFailedEvent: @@ -371,17 +383,18 @@ tests: - description: 'database.listCollectionNames retries at most maxAttempts=5 times' + operations: - name: failPoint object: testRunner arguments: - client: *failPointClient + client: *fail_point_client failPoint: configureFailPoint: failCommand mode: alwaysOn data: failCommands: [listCollections] - errorLabels: ["RetryableError", "SystemOverloadedError"] + errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - @@ -391,12 +404,13 @@ tests: filter: {} expectError: isError: true + isClientError: false expectEvents: - - client: "client" + - client: *client events: - # we expect 6 pairs of command started and succeeded events: 1 initial - # attempt and 5 retries. + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. - commandStartedEvent: commandName: listCollections - commandFailedEvent: @@ -425,17 +439,18 @@ tests: - description: 'database.runCommand retries at most maxAttempts=5 times' + operations: - name: failPoint object: testRunner arguments: - client: *failPointClient + client: *fail_point_client failPoint: configureFailPoint: failCommand mode: alwaysOn data: failCommands: [ping] - errorLabels: ["RetryableError", "SystemOverloadedError"] + errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - @@ -446,12 +461,13 @@ tests: commandName: ping expectError: isError: true + isClientError: false expectEvents: - - client: "client" + - client: *client events: - # we expect 6 pairs of command started and succeeded events: 1 initial - # attempt and 5 retries. + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. - commandStartedEvent: commandName: ping - commandFailedEvent: @@ -480,17 +496,18 @@ tests: - description: 'database.createChangeStream retries at most maxAttempts=5 times' + operations: - name: failPoint object: testRunner arguments: - client: *failPointClient + client: *fail_point_client failPoint: configureFailPoint: failCommand mode: alwaysOn data: failCommands: [aggregate] - errorLabels: ["RetryableError", "SystemOverloadedError"] + errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - @@ -500,12 +517,13 @@ tests: pipeline: [] expectError: isError: true + isClientError: false expectEvents: - - client: "client" + - client: *client events: - # we expect 6 pairs of command started and succeeded events: 1 initial - # attempt and 5 retries. + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. - commandStartedEvent: commandName: aggregate - commandFailedEvent: @@ -534,17 +552,18 @@ tests: - description: 'collection.aggregate retries at most maxAttempts=5 times' + operations: - name: failPoint object: testRunner arguments: - client: *failPointClient + client: *fail_point_client failPoint: configureFailPoint: failCommand mode: alwaysOn data: failCommands: [aggregate] - errorLabels: ["RetryableError", "SystemOverloadedError"] + errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - @@ -554,12 +573,13 @@ tests: pipeline: [] expectError: isError: true + isClientError: false expectEvents: - - client: "client" + - client: *client events: - # we expect 6 pairs of command started and succeeded events: 1 initial - # attempt and 5 retries. + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. - commandStartedEvent: commandName: aggregate - commandFailedEvent: @@ -588,17 +608,18 @@ tests: - description: 'collection.countDocuments retries at most maxAttempts=5 times' + operations: - name: failPoint object: testRunner arguments: - client: *failPointClient + client: *fail_point_client failPoint: configureFailPoint: failCommand mode: alwaysOn data: failCommands: [aggregate] - errorLabels: ["RetryableError", "SystemOverloadedError"] + errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - @@ -608,12 +629,13 @@ tests: filter: {} expectError: isError: true + isClientError: false expectEvents: - - client: "client" + - client: *client events: - # we expect 6 pairs of command started and succeeded events: 1 initial - # attempt and 5 retries. + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. - commandStartedEvent: commandName: aggregate - commandFailedEvent: @@ -642,17 +664,18 @@ tests: - description: 'collection.estimatedDocumentCount retries at most maxAttempts=5 times' + operations: - name: failPoint object: testRunner arguments: - client: *failPointClient + client: *fail_point_client failPoint: configureFailPoint: failCommand mode: alwaysOn data: failCommands: [count] - errorLabels: ["RetryableError", "SystemOverloadedError"] + errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - @@ -660,12 +683,13 @@ tests: name: estimatedDocumentCount expectError: isError: true + isClientError: false expectEvents: - - client: "client" + - client: *client events: - # we expect 6 pairs of command started and succeeded events: 1 initial - # attempt and 5 retries. + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. - commandStartedEvent: commandName: count - commandFailedEvent: @@ -694,17 +718,18 @@ tests: - description: 'collection.distinct retries at most maxAttempts=5 times' + operations: - name: failPoint object: testRunner arguments: - client: *failPointClient + client: *fail_point_client failPoint: configureFailPoint: failCommand mode: alwaysOn data: failCommands: [distinct] - errorLabels: ["RetryableError", "SystemOverloadedError"] + errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - @@ -715,12 +740,13 @@ tests: filter: {} expectError: isError: true + isClientError: false expectEvents: - - client: "client" + - client: *client events: - # we expect 6 pairs of command started and succeeded events: 1 initial - # attempt and 5 retries. + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. - commandStartedEvent: commandName: distinct - commandFailedEvent: @@ -749,17 +775,18 @@ tests: - description: 'collection.find retries at most maxAttempts=5 times' + operations: - name: failPoint object: testRunner arguments: - client: *failPointClient + client: *fail_point_client failPoint: configureFailPoint: failCommand mode: alwaysOn data: failCommands: [find] - errorLabels: ["RetryableError", "SystemOverloadedError"] + errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - @@ -769,12 +796,13 @@ tests: filter: {} expectError: isError: true + isClientError: false expectEvents: - - client: "client" + - client: *client events: - # we expect 6 pairs of command started and succeeded events: 1 initial - # attempt and 5 retries. + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. - commandStartedEvent: commandName: find - commandFailedEvent: @@ -803,17 +831,18 @@ tests: - description: 'collection.findOne retries at most maxAttempts=5 times' + operations: - name: failPoint object: testRunner arguments: - client: *failPointClient + client: *fail_point_client failPoint: configureFailPoint: failCommand mode: alwaysOn data: failCommands: [find] - errorLabels: ["RetryableError", "SystemOverloadedError"] + errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - @@ -823,12 +852,13 @@ tests: filter: {} expectError: isError: true + isClientError: false expectEvents: - - client: "client" + - client: *client events: - # we expect 6 pairs of command started and succeeded events: 1 initial - # attempt and 5 retries. + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. - commandStartedEvent: commandName: find - commandFailedEvent: @@ -857,17 +887,18 @@ tests: - description: 'collection.listIndexes retries at most maxAttempts=5 times' + operations: - name: failPoint object: testRunner arguments: - client: *failPointClient + client: *fail_point_client failPoint: configureFailPoint: failCommand mode: alwaysOn data: failCommands: [listIndexes] - errorLabels: ["RetryableError", "SystemOverloadedError"] + errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - @@ -875,12 +906,13 @@ tests: name: listIndexes expectError: isError: true + isClientError: false expectEvents: - - client: "client" + - client: *client events: - # we expect 6 pairs of command started and succeeded events: 1 initial - # attempt and 5 retries. + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. - commandStartedEvent: commandName: listIndexes - commandFailedEvent: @@ -909,17 +941,18 @@ tests: - description: 'collection.listIndexNames retries at most maxAttempts=5 times' + operations: - name: failPoint object: testRunner arguments: - client: *failPointClient + client: *fail_point_client failPoint: configureFailPoint: failCommand mode: alwaysOn data: failCommands: [listIndexes] - errorLabels: ["RetryableError", "SystemOverloadedError"] + errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - @@ -927,12 +960,13 @@ tests: name: listIndexNames expectError: isError: true + isClientError: false expectEvents: - - client: "client" + - client: *client events: - # we expect 6 pairs of command started and succeeded events: 1 initial - # attempt and 5 retries. + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. - commandStartedEvent: commandName: listIndexes - commandFailedEvent: @@ -961,17 +995,18 @@ tests: - description: 'collection.createChangeStream retries at most maxAttempts=5 times' + operations: - name: failPoint object: testRunner arguments: - client: *failPointClient + client: *fail_point_client failPoint: configureFailPoint: failCommand mode: alwaysOn data: failCommands: [aggregate] - errorLabels: ["RetryableError", "SystemOverloadedError"] + errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - @@ -981,12 +1016,13 @@ tests: pipeline: [] expectError: isError: true + isClientError: false expectEvents: - - client: "client" + - client: *client events: - # we expect 6 pairs of command started and succeeded events: 1 initial - # attempt and 5 retries. + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. - commandStartedEvent: commandName: aggregate - commandFailedEvent: @@ -1015,17 +1051,18 @@ tests: - description: 'collection.insertOne retries at most maxAttempts=5 times' + operations: - name: failPoint object: testRunner arguments: - client: *failPointClient + client: *fail_point_client failPoint: configureFailPoint: failCommand mode: alwaysOn data: failCommands: [insert] - errorLabels: ["RetryableError", "SystemOverloadedError"] + errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - @@ -1035,12 +1072,13 @@ tests: document: { _id: 2, x: 22 } expectError: isError: true + isClientError: false expectEvents: - - client: "client" + - client: *client events: - # we expect 6 pairs of command started and succeeded events: 1 initial - # attempt and 5 retries. + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. - commandStartedEvent: commandName: insert - commandFailedEvent: @@ -1069,17 +1107,18 @@ tests: - description: 'collection.insertMany retries at most maxAttempts=5 times' + operations: - name: failPoint object: testRunner arguments: - client: *failPointClient + client: *fail_point_client failPoint: configureFailPoint: failCommand mode: alwaysOn data: failCommands: [insert] - errorLabels: ["RetryableError", "SystemOverloadedError"] + errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - @@ -1090,12 +1129,13 @@ tests: - { _id: 2, x: 22 } expectError: isError: true + isClientError: false expectEvents: - - client: "client" + - client: *client events: - # we expect 6 pairs of command started and succeeded events: 1 initial - # attempt and 5 retries. + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. - commandStartedEvent: commandName: insert - commandFailedEvent: @@ -1124,17 +1164,18 @@ tests: - description: 'collection.deleteOne retries at most maxAttempts=5 times' + operations: - name: failPoint object: testRunner arguments: - client: *failPointClient + client: *fail_point_client failPoint: configureFailPoint: failCommand mode: alwaysOn data: failCommands: [delete] - errorLabels: ["RetryableError", "SystemOverloadedError"] + errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - @@ -1144,12 +1185,13 @@ tests: filter: {} expectError: isError: true + isClientError: false expectEvents: - - client: "client" + - client: *client events: - # we expect 6 pairs of command started and succeeded events: 1 initial - # attempt and 5 retries. + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. - commandStartedEvent: commandName: delete - commandFailedEvent: @@ -1178,17 +1220,18 @@ tests: - description: 'collection.deleteMany retries at most maxAttempts=5 times' + operations: - name: failPoint object: testRunner arguments: - client: *failPointClient + client: *fail_point_client failPoint: configureFailPoint: failCommand mode: alwaysOn data: failCommands: [delete] - errorLabels: ["RetryableError", "SystemOverloadedError"] + errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - @@ -1198,12 +1241,13 @@ tests: filter: {} expectError: isError: true + isClientError: false expectEvents: - - client: "client" + - client: *client events: - # we expect 6 pairs of command started and succeeded events: 1 initial - # attempt and 5 retries. + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. - commandStartedEvent: commandName: delete - commandFailedEvent: @@ -1232,17 +1276,18 @@ tests: - description: 'collection.replaceOne retries at most maxAttempts=5 times' + operations: - name: failPoint object: testRunner arguments: - client: *failPointClient + client: *fail_point_client failPoint: configureFailPoint: failCommand mode: alwaysOn data: failCommands: [update] - errorLabels: ["RetryableError", "SystemOverloadedError"] + errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - @@ -1253,12 +1298,13 @@ tests: replacement: { x: 22 } expectError: isError: true + isClientError: false expectEvents: - - client: "client" + - client: *client events: - # we expect 6 pairs of command started and succeeded events: 1 initial - # attempt and 5 retries. + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. - commandStartedEvent: commandName: update - commandFailedEvent: @@ -1287,17 +1333,18 @@ tests: - description: 'collection.updateOne retries at most maxAttempts=5 times' + operations: - name: failPoint object: testRunner arguments: - client: *failPointClient + client: *fail_point_client failPoint: configureFailPoint: failCommand mode: alwaysOn data: failCommands: [update] - errorLabels: ["RetryableError", "SystemOverloadedError"] + errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - @@ -1308,12 +1355,13 @@ tests: update: { $set: { x: 22 } } expectError: isError: true + isClientError: false expectEvents: - - client: "client" + - client: *client events: - # we expect 6 pairs of command started and succeeded events: 1 initial - # attempt and 5 retries. + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. - commandStartedEvent: commandName: update - commandFailedEvent: @@ -1342,17 +1390,18 @@ tests: - description: 'collection.updateMany retries at most maxAttempts=5 times' + operations: - name: failPoint object: testRunner arguments: - client: *failPointClient + client: *fail_point_client failPoint: configureFailPoint: failCommand mode: alwaysOn data: failCommands: [update] - errorLabels: ["RetryableError", "SystemOverloadedError"] + errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - @@ -1363,12 +1412,13 @@ tests: update: { $set: { x: 22 } } expectError: isError: true + isClientError: false expectEvents: - - client: "client" + - client: *client events: - # we expect 6 pairs of command started and succeeded events: 1 initial - # attempt and 5 retries. + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. - commandStartedEvent: commandName: update - commandFailedEvent: @@ -1397,17 +1447,18 @@ tests: - description: 'collection.findOneAndDelete retries at most maxAttempts=5 times' + operations: - name: failPoint object: testRunner arguments: - client: *failPointClient + client: *fail_point_client failPoint: configureFailPoint: failCommand mode: alwaysOn data: failCommands: [findAndModify] - errorLabels: ["RetryableError", "SystemOverloadedError"] + errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - @@ -1417,12 +1468,13 @@ tests: filter: {} expectError: isError: true + isClientError: false expectEvents: - - client: "client" + - client: *client events: - # we expect 6 pairs of command started and succeeded events: 1 initial - # attempt and 5 retries. + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. - commandStartedEvent: commandName: findAndModify - commandFailedEvent: @@ -1451,17 +1503,18 @@ tests: - description: 'collection.findOneAndReplace retries at most maxAttempts=5 times' + operations: - name: failPoint object: testRunner arguments: - client: *failPointClient + client: *fail_point_client failPoint: configureFailPoint: failCommand mode: alwaysOn data: failCommands: [findAndModify] - errorLabels: ["RetryableError", "SystemOverloadedError"] + errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - @@ -1472,12 +1525,13 @@ tests: replacement: { x: 22 } expectError: isError: true + isClientError: false expectEvents: - - client: "client" + - client: *client events: - # we expect 6 pairs of command started and succeeded events: 1 initial - # attempt and 5 retries. + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. - commandStartedEvent: commandName: findAndModify - commandFailedEvent: @@ -1506,17 +1560,18 @@ tests: - description: 'collection.findOneAndUpdate retries at most maxAttempts=5 times' + operations: - name: failPoint object: testRunner arguments: - client: *failPointClient + client: *fail_point_client failPoint: configureFailPoint: failCommand mode: alwaysOn data: failCommands: [findAndModify] - errorLabels: ["RetryableError", "SystemOverloadedError"] + errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - @@ -1527,12 +1582,13 @@ tests: update: { $set: { x: 22 } } expectError: isError: true + isClientError: false expectEvents: - - client: "client" + - client: *client events: - # we expect 6 pairs of command started and succeeded events: 1 initial - # attempt and 5 retries. + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. - commandStartedEvent: commandName: findAndModify - commandFailedEvent: @@ -1561,17 +1617,18 @@ tests: - description: 'collection.bulkWrite retries at most maxAttempts=5 times' + operations: - name: failPoint object: testRunner arguments: - client: *failPointClient + client: *fail_point_client failPoint: configureFailPoint: failCommand mode: alwaysOn data: failCommands: [insert] - errorLabels: ["RetryableError", "SystemOverloadedError"] + errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - @@ -1583,12 +1640,13 @@ tests: document: { _id: 2, x: 22 } expectError: isError: true + isClientError: false expectEvents: - - client: "client" + - client: *client events: - # we expect 6 pairs of command started and succeeded events: 1 initial - # attempt and 5 retries. + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. - commandStartedEvent: commandName: insert - commandFailedEvent: @@ -1617,17 +1675,18 @@ tests: - description: 'collection.createIndex retries at most maxAttempts=5 times' + operations: - name: failPoint object: testRunner arguments: - client: *failPointClient + client: *fail_point_client failPoint: configureFailPoint: failCommand mode: alwaysOn data: failCommands: [createIndexes] - errorLabels: ["RetryableError", "SystemOverloadedError"] + errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - @@ -1638,12 +1697,13 @@ tests: name: "x_11" expectError: isError: true + isClientError: false expectEvents: - - client: "client" + - client: *client events: - # we expect 6 pairs of command started and succeeded events: 1 initial - # attempt and 5 retries. + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. - commandStartedEvent: commandName: createIndexes - commandFailedEvent: @@ -1672,17 +1732,18 @@ tests: - description: 'collection.dropIndex retries at most maxAttempts=5 times' + operations: - name: failPoint object: testRunner arguments: - client: *failPointClient + client: *fail_point_client failPoint: configureFailPoint: failCommand mode: alwaysOn data: failCommands: [dropIndexes] - errorLabels: ["RetryableError", "SystemOverloadedError"] + errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - @@ -1692,12 +1753,13 @@ tests: name: "x_11" expectError: isError: true + isClientError: false expectEvents: - - client: "client" + - client: *client events: - # we expect 6 pairs of command started and succeeded events: 1 initial - # attempt and 5 retries. + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. - commandStartedEvent: commandName: dropIndexes - commandFailedEvent: @@ -1726,17 +1788,18 @@ tests: - description: 'collection.dropIndexes retries at most maxAttempts=5 times' + operations: - name: failPoint object: testRunner arguments: - client: *failPointClient + client: *fail_point_client failPoint: configureFailPoint: failCommand mode: alwaysOn data: failCommands: [dropIndexes] - errorLabels: ["RetryableError", "SystemOverloadedError"] + errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - @@ -1744,12 +1807,13 @@ tests: name: dropIndexes expectError: isError: true + isClientError: false expectEvents: - - client: "client" + - client: *client events: - # we expect 6 pairs of command started and succeeded events: 1 initial - # attempt and 5 retries. + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. - commandStartedEvent: commandName: dropIndexes - commandFailedEvent: diff --git a/source/client-backpressure/tests/backpressure-retry-max-attempts.yml.template b/source/client-backpressure/tests/backpressure-retry-max-attempts.yml.template index 4f2cfeee47..9efbdfff19 100644 --- a/source/client-backpressure/tests/backpressure-retry-max-attempts.yml.template +++ b/source/client-backpressure/tests/backpressure-retry-max-attempts.yml.template @@ -14,11 +14,11 @@ createEntities: client: id: &client client useMultipleMongoses: false - observeEvents: [ 'commandStartedEvent', 'commandSucceededEvent', 'commandFailedEvent' ] + observeEvents: [commandStartedEvent, commandSucceededEvent, commandFailedEvent] - client: - id: &failPointClient failPointClient + id: &fail_point_client fail_point_client useMultipleMongoses: false - @@ -48,17 +48,18 @@ tests: runOnRequirements: - minServerVersion: '8.0' # client bulk write added to server in 8.0 {%- endif %} + operations: - name: failPoint object: testRunner arguments: - client: *failPointClient + client: *fail_point_client failPoint: configureFailPoint: failCommand mode: alwaysOn data: failCommands: [{{operation.command_name}}] - errorLabels: ["RetryableError", "SystemOverloadedError"] + errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - @@ -72,12 +73,13 @@ tests: {%- endif %} expectError: isError: true + isClientError: false expectEvents: - - client: "client" + - client: *client events: - # we expect 6 pairs of command started and succeeded events: 1 initial - # attempt and 5 retries. + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. - commandStartedEvent: commandName: {{operation.command_name}} - commandFailedEvent: