diff --git a/source/client-side-encryption/limits/limits-encryptedFields.json b/source/client-side-encryption/limits/limits-encryptedFields.json new file mode 100644 index 0000000000..c52a0271e1 --- /dev/null +++ b/source/client-side-encryption/limits/limits-encryptedFields.json @@ -0,0 +1,14 @@ +{ + "fields": [ + { + "keyId": { + "$binary": { + "base64": "LOCALAAAAAAAAAAAAAAAAA==", + "subType": "04" + } + }, + "path": "foo", + "bsonType": "string" + } + ] +} \ No newline at end of file diff --git a/source/client-side-encryption/limits/limits-qe-doc.json b/source/client-side-encryption/limits/limits-qe-doc.json new file mode 100644 index 0000000000..71efbf4068 --- /dev/null +++ b/source/client-side-encryption/limits/limits-qe-doc.json @@ -0,0 +1,3 @@ +{ + "foo": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" +} \ No newline at end of file diff --git a/source/client-side-encryption/tests/README.md b/source/client-side-encryption/tests/README.md index e0677bbab7..6deb4f81a3 100644 --- a/source/client-side-encryption/tests/README.md +++ b/source/client-side-encryption/tests/README.md @@ -563,10 +563,13 @@ First, perform the setup. 2. Using `client`, drop and create the collection `db.coll` configured with the included JSON schema [limits/limits-schema.json](../limits/limits-schema.json). -3. Using `client`, drop the collection `keyvault.datakeys`. Insert the document +3. If using MongoDB 8.0+, use `client` to drop and create the collection `db.coll2` configured with the included + encryptedFields [limits/limits-encryptedFields.json](../limits/limits-encryptedFields.json). + +4. Using `client`, drop the collection `keyvault.datakeys`. Insert the document [limits/limits-key.json](../limits/limits-key.json) -4. Create a MongoClient configured with auto encryption (referred to as `client_encrypted`) +5. Create a MongoClient configured with auto encryption (referred to as `client_encrypted`) Configure with the `local` KMS provider as follows: @@ -578,19 +581,19 @@ First, perform the setup. Using `client_encrypted` perform the following operations: -1. Insert `{ "_id": "over_2mib_under_16mib", "unencrypted": }`. +1. Insert `{ "_id": "over_2mib_under_16mib", "unencrypted": }` into `coll`. Expect this to succeed since this is still under the `maxBsonObjectSize` limit. 2. Insert the document [limits/limits-doc.json](../limits/limits-doc.json) concatenated with - `{ "_id": "encryption_exceeds_2mib", "unencrypted": < the string "a" repeated (2097152 - 2000) times > }` Note: - limits-doc.json is a 1005 byte BSON document that encrypts to a ~10,000 byte document. + `{ "_id": "encryption_exceeds_2mib", "unencrypted": < the string "a" repeated (2097152 - 2000) times > }` into + `coll`. Note: limits-doc.json is a 1005 byte BSON document that encrypts to a ~10,000 byte document. Expect this to succeed since after encryption this still is below the normal maximum BSON document size. Note, before auto encryption this document is under the 2 MiB limit. After encryption it exceeds the 2 MiB limit, but does NOT exceed the 16 MiB limit. -3. Bulk insert the following: +3. Use MongoCollection.bulkWrite to insert the following into `coll`: - `{ "_id": "over_2mib_1", "unencrypted": }` - `{ "_id": "over_2mib_2", "unencrypted": }` @@ -598,7 +601,7 @@ Using `client_encrypted` perform the following operations: Expect the bulk write to succeed and split after first doc (i.e. two inserts occur). This may be verified using [command monitoring](../../command-logging-and-monitoring/command-logging-and-monitoring.md). -4. Bulk insert the following: +4. Use MongoCollection.bulkWrite insert the following into `coll`: - The document [limits/limits-doc.json](../limits/limits-doc.json) concatenated with `{ "_id": "encryption_exceeds_2mib_1", "unencrypted": < the string "a" repeated (2097152 - 2000) times > }` @@ -608,15 +611,34 @@ Using `client_encrypted` perform the following operations: Expect the bulk write to succeed and split after first doc (i.e. two inserts occur). This may be verified using [command logging and monitoring](../../command-logging-and-monitoring/command-logging-and-monitoring.md). -5. Insert `{ "_id": "under_16mib", "unencrypted": `. +5. Insert `{ "_id": "under_16mib", "unencrypted": ` into `coll`. Expect this to succeed since this is still (just) under the `maxBsonObjectSize` limit. 6. Insert the document [limits/limits-doc.json](../limits/limits-doc.json) concatenated with - `{ "_id": "encryption_exceeds_16mib", "unencrypted": < the string "a" repeated (16777216 - 2000) times > }` + `{ "_id": "encryption_exceeds_16mib", "unencrypted": < the string "a" repeated (16777216 - 2000) times > }` into + `coll`. Expect this to fail since encryption results in a document exceeding the `maxBsonObjectSize` limit. +7. If using MongoDB 8.0+, use MongoClient.bulkWrite to insert the following into `coll2`: + + - `{ "_id": "over_2mib_3", "unencrypted": }` + - `{ "_id": "over_2mib_4", "unencrypted": }` + + Expect the bulk write to succeed and split after first doc (i.e. two inserts occur). This may be verified using + [command logging and monitoring](../../command-logging-and-monitoring/command-logging-and-monitoring.md). + +8. If using MongoDB 8.0+, use MongoClient.bulkWrite to insert the following into `coll2`: + + - The document [limits/limits-qe-doc.json](../limits/limits-qe-doc.json) concatenated with + `{ "_id": "encryption_exceeds_2mib_3", "foo": < the string "a" repeated (2097152 - 2000 - 1500) times > }` + - The document [limits/limits-qe-doc.json](../limits/limits-qe-doc.json) concatenated with + `{ "_id": "encryption_exceeds_2mib_4", "foo": < the string "a" repeated (2097152 - 2000 - 1500) times > }` + + Expect the bulk write to succeed and split after first doc (i.e. two inserts occur). This may be verified using + [command logging and monitoring](../../command-logging-and-monitoring/command-logging-and-monitoring.md). + Optionally, if it is possible to mock the maxWriteBatchSize (i.e. the maximum number of documents in a batch) test that setting maxWriteBatchSize=1 and inserting the two documents `{ "_id": "a" }, { "_id": "b" }` with `client_encrypted` splits the operation into two inserts. diff --git a/source/crud/bulk-write.md b/source/crud/bulk-write.md index 28c5b847fd..b07329ef22 100644 --- a/source/crud/bulk-write.md +++ b/source/crud/bulk-write.md @@ -459,7 +459,7 @@ class BulkWriteResult { * The results of each individual write operation that was successfully performed. * * This value will only be populated if the verboseResults option was set to true. - */ + */ verboseResults: Optional; /* rest of fields */ @@ -553,7 +553,9 @@ The `bulkWrite` server command has the following format: } ``` -Drivers MUST use document sequences ([`OP_MSG`](../message/OP_MSG.md) payload type 1) for the `ops` and `nsInfo` fields. +If auto-encryption is not enabled, drivers MUST use document sequences ([`OP_MSG`](../message/OP_MSG.md) payload type 1) +for the `ops` and `nsInfo` fields. If auto-encryption is enabled, drivers MUST NOT use document sequences and MUST +append the `ops` and `nsInfo` fields to the `bulkWrite` command document. The `bulkWrite` command is executed on the "admin" database. @@ -645,13 +647,6 @@ write concern containing the following message: > Cannot request unacknowledged write concern and ordered writes -## Auto-Encryption - -If `MongoClient.bulkWrite` is called on a `MongoClient` configured with `AutoEncryptionOpts`, drivers MUST return an -error with the message: "bulkWrite does not currently support automatic encryption". - -This is expected to be removed once [DRIVERS-2888](https://jira.mongodb.org/browse/DRIVERS-2888) is implemented. - ## Command Batching Drivers MUST accept an arbitrary number of operations as input to the `MongoClient.bulkWrite` method. Because the server @@ -672,8 +667,10 @@ multiple commands if the user provides more than `maxWriteBatchSize` operations ### Total Message Size -Drivers MUST ensure that the total size of the `OP_MSG` built for each `bulkWrite` command does not exceed -`maxMessageSizeBytes`. +#### Unencrypted bulk writes + +When auto-encryption is not enabled, drivers MUST ensure that the total size of the `OP_MSG` built for each `bulkWrite` +command does not exceed `maxMessageSizeBytes`. The upper bound for the size of an `OP_MSG` includes opcode-related bytes (e.g. the `OP_MSG` header) and operation-agnostic command field bytes (e.g. `txnNumber`, `lsid`). Drivers MUST limit the combined size of the @@ -727,6 +724,12 @@ was determined. Drivers MUST return an error if there is not room to add at least one operation to `ops`. +#### Auto-encrypted bulk writes + +Drivers MUST use the reduced size limit defined in +[Size limits for Write Commands](../client-side-encryption/client-side-encryption.md#size-limits-for-write-commands) for +the size of the `bulkWrite` command document when auto-encryption is enabled. + ## Handling the `bulkWrite` Server Response The server's response to `bulkWrite` has the following format: @@ -857,6 +860,15 @@ When a `getMore` fails with a retryable error when attempting to iterate the res entire `bulkWrite` command to receive a fresh cursor and retry iteration. This work was omitted to minimize the scope of the initial implementation and testing of the new bulk write API, but may be revisited in the future. +### Use document sequences for auto-encrypted bulk writes + +Auto-encryption does not currently support document sequences. This specification should be updated when +[DRIVERS-2859](https://jira.mongodb.org/browse/DRIVERS-2859) is completed to require use of document sequences for `ops` +and `nsInfo` when auto-encryption is enabled. + +Drivers requiring significant changes to pass a bulkWrite command to libmongocrypt are recommended to wait until +[DRIVERS-2859](https://jira.mongodb.org/browse/DRIVERS-2859) is implemented before supporting automatic encryption. + ## Q&A ### Is `bulkWrite` supported on Atlas Serverless? @@ -928,6 +940,8 @@ error in this specific situation does not seem helpful enough to require size ch ## **Changelog** +- 2025-08-13: Removed the requirement to error when QE is enabled. + - 2025-06-27: Added `rawData` option. - 2024-11-05: Updated the requirements regarding the size validation. diff --git a/source/crud/tests/README.md b/source/crud/tests/README.md index 92365a6f05..465fff7db6 100644 --- a/source/crud/tests/README.md +++ b/source/crud/tests/README.md @@ -607,40 +607,7 @@ Execute `bulkWrite` on `client` with `largeNamespaceModel`. Assert that an error Assert that `error` is a client error. If a `BulkWriteException` was thrown, assert `BulkWriteException.partialResult` is unset. -### 13. `MongoClient.bulkWrite` returns an error if auto-encryption is configured - -This test is expected to be removed when [DRIVERS-2888](https://jira.mongodb.org/browse/DRIVERS-2888) is resolved. - -Test that `MongoClient.bulkWrite` returns an error if the client has auto-encryption configured. - -This test must only be run on 8.0+ servers. This test must be skipped on Atlas Serverless. - -Construct a `MongoClient` (referred to as `client`) configured with the following `AutoEncryptionOpts`: - -```javascript -AutoEncryptionOpts { - "keyVaultNamespace": "db.coll", - "kmsProviders": { - "aws": { - "accessKeyId": "foo", - "secretAccessKey": "bar" - } - } -} -``` - -Construct the following write model (referred to as `model`): - -```javascript -InsertOne { - "namespace": "db.coll", - "document": { "a": "b" } -} -``` - -Execute `bulkWrite` on `client` with `model`. Assert that an error (referred to as `error`) is returned. Assert that -`error` is a client error containing the message: "bulkWrite does not currently support automatic encryption". If a -`BulkWriteException` was thrown, assert `BulkWriteException.partialResult` is unset. +### 13. *Removed* ### 14. `explain` helpers allow users to specify `maxTimeMS` diff --git a/source/crud/tests/unified/client-bulkWrite-qe.json b/source/crud/tests/unified/client-bulkWrite-qe.json new file mode 100644 index 0000000000..9ed46025a6 --- /dev/null +++ b/source/crud/tests/unified/client-bulkWrite-qe.json @@ -0,0 +1,305 @@ +{ + "description": "client bulkWrite with queryable encryption", + "schemaVersion": "1.25", + "runOnRequirements": [ + { + "minServerVersion": "8.0", + "topologies": [ + "replicaset", + "sharded", + "load-balanced" + ], + "serverless": "forbid", + "csfle": { + "minLibmongocryptVersion": "1.10.0" + } + } + ], + "createEntities": [ + { + "client": { + "id": "client0", + "observeEvents": [ + "commandStartedEvent", + "commandSucceededEvent" + ], + "autoEncryptOpts": { + "keyVaultNamespace": "keyvault.datakeys", + "kmsProviders": { + "local": { + "key": "Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk" + } + } + } + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "crud-tests" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "coll0" + } + }, + { + "client": { + "id": "client1", + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "database": { + "id": "database1", + "client": "client0", + "databaseName": "keyvault" + } + }, + { + "collection": { + "id": "collection1", + "database": "database0", + "collectionName": "datakeys" + } + }, + { + "database": { + "id": "database2", + "client": "client1", + "databaseName": "crud-tests" + } + }, + { + "collection": { + "id": "collection2", + "database": "database2", + "collectionName": "coll0" + } + } + ], + "initialData": [ + { + "databaseName": "keyvault", + "collectionName": "datakeys", + "documents": [ + { + "_id": { + "$binary": { + "base64": "EjRWeBI0mHYSNBI0VniQEg==", + "subType": "04" + } + }, + "keyAltNames": [ + "local_key" + ], + "keyMaterial": { + "$binary": { + "base64": "sHe0kz57YW7v8g9VP9sf/+K1ex4JqKc5rf/URX3n3p8XdZ6+15uXPaSayC6adWbNxkFskuMCOifDoTT+rkqMtFkDclOy884RuGGtUysq3X7zkAWYTKi8QAfKkajvVbZl2y23UqgVasdQu3OVBQCrH/xY00nNAs/52e958nVjBuzQkSb1T8pKJAyjZsHJ60+FtnfafDZSTAIBJYn7UWBCwQ==", + "subType": "00" + } + }, + "creationDate": { + "$date": { + "$numberLong": "1641024000000" + } + }, + "updateDate": { + "$date": { + "$numberLong": "1641024000000" + } + }, + "status": 1, + "masterKey": { + "provider": "local" + } + } + ] + }, + { + "databaseName": "crud-tests", + "collectionName": "coll0", + "documents": [], + "createOptions": { + "encryptedFields": { + "fields": [ + { + "keyId": { + "$binary": { + "base64": "EjRWeBI0mHYSNBI0VniQEg==", + "subType": "04" + } + }, + "path": "encryptedInt", + "bsonType": "int", + "queries": { + "queryType": "equality", + "contention": { + "$numberLong": "0" + } + } + } + ] + } + } + } + ], + "_yamlAnchors": { + "namespace": "crud-tests.coll0" + }, + "tests": [ + { + "description": "client bulkWrite QE replaceOne", + "operations": [ + { + "object": "collection0", + "name": "insertMany", + "arguments": { + "documents": [ + { + "_id": 1, + "encryptedInt": 11 + }, + { + "_id": 2, + "encryptedInt": 22 + }, + { + "_id": 3, + "encryptedInt": 33 + } + ] + } + }, + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "models": [ + { + "replaceOne": { + "namespace": "crud-tests.coll0", + "filter": { + "encryptedInt": { + "$eq": 11 + } + }, + "replacement": { + "encryptedInt": 44 + } + } + } + ] + }, + "expectResult": { + "insertedCount": 0, + "upsertedCount": 0, + "matchedCount": 1, + "modifiedCount": 1, + "deletedCount": 0 + } + }, + { + "object": "collection0", + "name": "find", + "arguments": { + "filter": { + "encryptedInt": 44 + } + }, + "expectResult": [ + { + "_id": 1, + "encryptedInt": 44 + } + ] + }, + { + "object": "collection2", + "name": "find", + "arguments": { + "filter": {} + }, + "expectResult": [ + { + "_id": 1, + "encryptedInt": { + "$$type": "binData" + }, + "__safeContent__": { + "$$type": "array" + } + }, + { + "_id": 2, + "encryptedInt": { + "$$type": "binData" + }, + "__safeContent__": { + "$$type": "array" + } + }, + { + "_id": 3, + "encryptedInt": { + "$$type": "binData" + }, + "__safeContent__": { + "$$type": "array" + } + } + ] + } + ] + }, + { + "description": "client bulkWrite QE with multiple replace fails", + "operations": [ + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "models": [ + { + "replaceOne": { + "namespace": "crud-tests.coll0", + "filter": { + "encryptedInt": { + "$eq": 11 + } + }, + "replacement": { + "encryptedInt": 44 + } + } + }, + { + "replaceOne": { + "namespace": "crud-tests.coll0", + "filter": { + "encryptedInt": { + "$eq": 22 + } + }, + "replacement": { + "encryptedInt": 44 + } + } + } + ] + }, + "expectError": { + "isError": true, + "errorContains": "Only insert is supported in BulkWrite with multiple operations and Queryable Encryption" + } + } + ] + } + ] +} diff --git a/source/crud/tests/unified/client-bulkWrite-qe.yml b/source/crud/tests/unified/client-bulkWrite-qe.yml new file mode 100644 index 0000000000..1c7adadae0 --- /dev/null +++ b/source/crud/tests/unified/client-bulkWrite-qe.yml @@ -0,0 +1,130 @@ +description: client bulkWrite with queryable encryption + +schemaVersion: "1.25" + +runOnRequirements: + - minServerVersion: "8.0" + topologies: ["replicaset", "sharded", "load-balanced"] # QE does not support standalone. + serverless: forbid # Serverless does not support bulkWrite: CLOUDP-256344. + csfle: + minLibmongocryptVersion: "1.10.0" + +createEntities: + - client: + id: &client0 client0 + observeEvents: + - commandStartedEvent + - commandSucceededEvent + autoEncryptOpts: + keyVaultNamespace: keyvault.datakeys + kmsProviders: + local: + key: Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk + - database: + id: &database0 database0 + client: *client0 + databaseName: &database0Name crud-tests + - collection: + id: &collection0 collection0 + database: *database0 + collectionName: &collection0Name coll0 + - client: + id: &client1 client1 + observeEvents: + - commandStartedEvent + - database: + id: &database1 database1 + client: *client0 + databaseName: &database1Name keyvault + - collection: + id: &collection1 collection1 + database: *database0 + collectionName: &collection1Name datakeys + - database: + id: &database2 database2 + client: *client1 + databaseName: &database0Name crud-tests + - collection: + id: &collection2 collection2 + database: *database2 + collectionName: &collection0Name coll0 + + +initialData: + - databaseName: *database1Name + collectionName: *collection1Name + documents: + - _id: &local_key_id { $binary: { base64: EjRWeBI0mHYSNBI0VniQEg==, subType: "04" } } + keyAltNames: ["local_key"] + keyMaterial: { $binary: { base64: sHe0kz57YW7v8g9VP9sf/+K1ex4JqKc5rf/URX3n3p8XdZ6+15uXPaSayC6adWbNxkFskuMCOifDoTT+rkqMtFkDclOy884RuGGtUysq3X7zkAWYTKi8QAfKkajvVbZl2y23UqgVasdQu3OVBQCrH/xY00nNAs/52e958nVjBuzQkSb1T8pKJAyjZsHJ60+FtnfafDZSTAIBJYn7UWBCwQ==, subType: "00" } } + creationDate: { $date: { $numberLong: "1641024000000" } } + updateDate: { $date: { $numberLong: "1641024000000" } } + status: 1 + masterKey: &local_masterkey + provider: local + - databaseName: *database0Name + collectionName: *collection0Name + documents: [] + createOptions: + encryptedFields: &encrypted_fields {'fields': [{'keyId': {'$binary': {'base64': 'EjRWeBI0mHYSNBI0VniQEg==', 'subType': '04'}}, 'path': 'encryptedInt', 'bsonType': 'int', 'queries': {'queryType': 'equality', 'contention': {'$numberLong': '0'}}}]} + +_yamlAnchors: + namespace: &namespace "crud-tests.coll0" + +tests: + - description: client bulkWrite QE replaceOne + operations: + - object: *collection0 + name: insertMany + arguments: + documents: + - { _id: 1, encryptedInt: 11 } + - { _id: 2, encryptedInt: 22 } + - { _id: 3, encryptedInt: 33 } + - object: *client0 + name: clientBulkWrite + arguments: + models: + - replaceOne: + namespace: *namespace + filter: { encryptedInt: { $eq: 11 } } + replacement: { encryptedInt: 44 } + expectResult: + insertedCount: 0 + upsertedCount: 0 + matchedCount: 1 + modifiedCount: 1 + deletedCount: 0 + - object: *collection0 + name: find + arguments: + filter: { encryptedInt: 44 } + expectResult: + - _id: 1 + encryptedInt: 44 + - object: *collection2 + name: find + arguments: + filter: {} + expectResult: + - { _id: 1, encryptedInt: { $$type: binData }, __safeContent__: { $$type: array} } + - { _id: 2, encryptedInt: { $$type: binData }, __safeContent__: { $$type: array} } + - { _id: 3, encryptedInt: { $$type: binData }, __safeContent__: { $$type: array} } + - description: client bulkWrite QE with multiple replace fails + operations: + - object: *client0 + name: clientBulkWrite + arguments: + models: + - replaceOne: + namespace: *namespace + filter: { encryptedInt: { $eq: 11 } } + replacement: { encryptedInt: 44 } + - replaceOne: + namespace: *namespace + filter: { encryptedInt: { $eq: 22 } } + replacement: { encryptedInt: 44 } + expectError: + # Expect error from mongocryptd or crypt_shared + isError: true + errorContains: "Only insert is supported in BulkWrite with multiple operations and Queryable Encryption"