diff --git a/src/libmongoc/src/mongoc/mongoc-bulkwrite.c b/src/libmongoc/src/mongoc/mongoc-bulkwrite.c index 95678bfe019..a590bc785ca 100644 --- a/src/libmongoc/src/mongoc/mongoc-bulkwrite.c +++ b/src/libmongoc/src/mongoc/mongoc-bulkwrite.c @@ -1569,15 +1569,6 @@ mongoc_bulkwrite_execute(mongoc_bulkwrite_t *self, const mongoc_bulkwriteopts_t goto fail; } - if (_mongoc_cse_is_enabled(self->client)) { - _mongoc_set_error(&error, - MONGOC_ERROR_COMMAND, - MONGOC_ERROR_COMMAND_INVALID_ARG, - "bulkWrite does not currently support automatic encryption"); - _bulkwriteexception_set_error(ret.exc, &error); - goto fail; - } - const mongoc_ss_log_context_t ss_log_context = { .operation = "bulkWrite", .has_operation_id = true, .operation_id = self->operation_id}; @@ -1703,8 +1694,11 @@ mongoc_bulkwrite_execute(mongoc_bulkwrite_t *self, const mongoc_bulkwriteopts_t } } - int32_t maxWriteBatchSize = mongoc_server_stream_max_write_batch_size(ss); + const int32_t maxWriteBatchSize = mongoc_server_stream_max_write_batch_size(ss); int32_t maxMessageSizeBytes = mongoc_server_stream_max_msg_size(ss); + if (_mongoc_cse_is_enabled(self->client)) { + maxMessageSizeBytes = MONGOC_REDUCED_MAX_MSG_SIZE_FOR_FLE; + } // `ops_doc_offset` is an offset into the `ops` document sequence. Counts the number of documents sent. size_t ops_doc_offset = 0; // `ops_byte_offset` is an offset into the `ops` document sequence. Counts the number of bytes sent. diff --git a/src/libmongoc/src/mongoc/mongoc-crypt.c b/src/libmongoc/src/mongoc/mongoc-crypt.c index 06a5a25e5b4..183badb41e0 100644 --- a/src/libmongoc/src/mongoc/mongoc-crypt.c +++ b/src/libmongoc/src/mongoc/mongoc-crypt.c @@ -319,9 +319,9 @@ _state_machine_destroy(_state_machine_t *state_machine) bson_free(state_machine); } -/* State handler MONGOCRYPT_CTX_NEED_MONGO_COLLINFO */ +/* State handler MONGOCRYPT_CTX_NEED_MONGO_COLLINFO{_WITH_DB} */ static bool -_state_need_mongo_collinfo(_state_machine_t *state_machine, bson_error_t *error) +_state_need_mongo_collinfo(_state_machine_t *state_machine, const char *db_name, bson_error_t *error) { mongoc_database_t *db = NULL; mongoc_cursor_t *cursor = NULL; @@ -345,7 +345,12 @@ _state_need_mongo_collinfo(_state_machine_t *state_machine, bson_error_t *error) } bson_append_document(&opts, "filter", -1, &filter_bson); - db = mongoc_client_get_database(state_machine->collinfo_client, state_machine->db_name); + if (!db_name) { + _ctx_check_error(state_machine->ctx, error, true); + goto fail; + } + db = mongoc_client_get_database(state_machine->collinfo_client, db_name); + cursor = mongoc_database_find_collections_with_opts(db, &opts); if (mongoc_cursor_error(cursor, error)) { goto fail; @@ -1085,7 +1090,7 @@ _state_machine_run(_state_machine_t *state_machine, bson_t *result, bson_error_t _ctx_check_error(state_machine->ctx, error, true); goto fail; case MONGOCRYPT_CTX_NEED_MONGO_COLLINFO: - if (!_state_need_mongo_collinfo(state_machine, error)) { + if (!_state_need_mongo_collinfo(state_machine, state_machine->db_name, error)) { goto fail; } break; @@ -1118,14 +1123,16 @@ _state_machine_run(_state_machine_t *state_machine, bson_t *result, bson_error_t case MONGOCRYPT_CTX_DONE: goto success; break; - case MONGOCRYPT_CTX_NEED_MONGO_COLLINFO_WITH_DB: - _mongoc_set_error(error, - MONGOC_ERROR_CLIENT_SIDE_ENCRYPTION, - MONGOC_ERROR_CLIENT_INVALID_ENCRYPTION_STATE, - "MONGOCRYPT_CTX_NEED_MONGO_COLLINFO_WITH_DB is " - "unimplemented"); - goto fail; - break; + case MONGOCRYPT_CTX_NEED_MONGO_COLLINFO_WITH_DB: { + const char *db_name = mongocrypt_ctx_mongo_db(state_machine->ctx); + if (!db_name) { + _ctx_check_error(state_machine->ctx, error, true); + goto fail; + } + if (!_state_need_mongo_collinfo(state_machine, db_name, error)) { + goto fail; + } + } break; } } @@ -1407,6 +1414,8 @@ _mongoc_crypt_new(const bson_t *kms_providers, crypt->kmsid_to_tlsopts = mcd_mapof_kmsid_to_tlsopts_new(); crypt->handle = mongocrypt_new(); mongocrypt_setopt_retry_kms(crypt->handle, true); + mongocrypt_setopt_use_need_mongo_collinfo_with_db_state(crypt->handle); + if (!mongocrypt_setopt_enable_multiple_collinfo(crypt->handle)) { _crypt_check_error(crypt->handle, error, true); goto fail; diff --git a/src/libmongoc/tests/client_side_encryption_prose/limits-encryptedFields.json b/src/libmongoc/tests/client_side_encryption_prose/limits-encryptedFields.json new file mode 100644 index 00000000000..c52a0271e16 --- /dev/null +++ b/src/libmongoc/tests/client_side_encryption_prose/limits-encryptedFields.json @@ -0,0 +1,14 @@ +{ + "fields": [ + { + "keyId": { + "$binary": { + "base64": "LOCALAAAAAAAAAAAAAAAAA==", + "subType": "04" + } + }, + "path": "foo", + "bsonType": "string" + } + ] +} \ No newline at end of file diff --git a/src/libmongoc/tests/client_side_encryption_prose/limits-qe-doc.json b/src/libmongoc/tests/client_side_encryption_prose/limits-qe-doc.json new file mode 100644 index 00000000000..71efbf40682 --- /dev/null +++ b/src/libmongoc/tests/client_side_encryption_prose/limits-qe-doc.json @@ -0,0 +1,3 @@ +{ + "foo": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" +} \ No newline at end of file diff --git a/src/libmongoc/tests/json/client_side_encryption/unified/client-bulkWrite-qe.json b/src/libmongoc/tests/json/client_side_encryption/unified/client-bulkWrite-qe.json new file mode 100644 index 00000000000..9ed46025a68 --- /dev/null +++ b/src/libmongoc/tests/json/client_side_encryption/unified/client-bulkWrite-qe.json @@ -0,0 +1,305 @@ +{ + "description": "client bulkWrite with queryable encryption", + "schemaVersion": "1.25", + "runOnRequirements": [ + { + "minServerVersion": "8.0", + "topologies": [ + "replicaset", + "sharded", + "load-balanced" + ], + "serverless": "forbid", + "csfle": { + "minLibmongocryptVersion": "1.10.0" + } + } + ], + "createEntities": [ + { + "client": { + "id": "client0", + "observeEvents": [ + "commandStartedEvent", + "commandSucceededEvent" + ], + "autoEncryptOpts": { + "keyVaultNamespace": "keyvault.datakeys", + "kmsProviders": { + "local": { + "key": "Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk" + } + } + } + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "crud-tests" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "coll0" + } + }, + { + "client": { + "id": "client1", + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "database": { + "id": "database1", + "client": "client0", + "databaseName": "keyvault" + } + }, + { + "collection": { + "id": "collection1", + "database": "database0", + "collectionName": "datakeys" + } + }, + { + "database": { + "id": "database2", + "client": "client1", + "databaseName": "crud-tests" + } + }, + { + "collection": { + "id": "collection2", + "database": "database2", + "collectionName": "coll0" + } + } + ], + "initialData": [ + { + "databaseName": "keyvault", + "collectionName": "datakeys", + "documents": [ + { + "_id": { + "$binary": { + "base64": "EjRWeBI0mHYSNBI0VniQEg==", + "subType": "04" + } + }, + "keyAltNames": [ + "local_key" + ], + "keyMaterial": { + "$binary": { + "base64": "sHe0kz57YW7v8g9VP9sf/+K1ex4JqKc5rf/URX3n3p8XdZ6+15uXPaSayC6adWbNxkFskuMCOifDoTT+rkqMtFkDclOy884RuGGtUysq3X7zkAWYTKi8QAfKkajvVbZl2y23UqgVasdQu3OVBQCrH/xY00nNAs/52e958nVjBuzQkSb1T8pKJAyjZsHJ60+FtnfafDZSTAIBJYn7UWBCwQ==", + "subType": "00" + } + }, + "creationDate": { + "$date": { + "$numberLong": "1641024000000" + } + }, + "updateDate": { + "$date": { + "$numberLong": "1641024000000" + } + }, + "status": 1, + "masterKey": { + "provider": "local" + } + } + ] + }, + { + "databaseName": "crud-tests", + "collectionName": "coll0", + "documents": [], + "createOptions": { + "encryptedFields": { + "fields": [ + { + "keyId": { + "$binary": { + "base64": "EjRWeBI0mHYSNBI0VniQEg==", + "subType": "04" + } + }, + "path": "encryptedInt", + "bsonType": "int", + "queries": { + "queryType": "equality", + "contention": { + "$numberLong": "0" + } + } + } + ] + } + } + } + ], + "_yamlAnchors": { + "namespace": "crud-tests.coll0" + }, + "tests": [ + { + "description": "client bulkWrite QE replaceOne", + "operations": [ + { + "object": "collection0", + "name": "insertMany", + "arguments": { + "documents": [ + { + "_id": 1, + "encryptedInt": 11 + }, + { + "_id": 2, + "encryptedInt": 22 + }, + { + "_id": 3, + "encryptedInt": 33 + } + ] + } + }, + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "models": [ + { + "replaceOne": { + "namespace": "crud-tests.coll0", + "filter": { + "encryptedInt": { + "$eq": 11 + } + }, + "replacement": { + "encryptedInt": 44 + } + } + } + ] + }, + "expectResult": { + "insertedCount": 0, + "upsertedCount": 0, + "matchedCount": 1, + "modifiedCount": 1, + "deletedCount": 0 + } + }, + { + "object": "collection0", + "name": "find", + "arguments": { + "filter": { + "encryptedInt": 44 + } + }, + "expectResult": [ + { + "_id": 1, + "encryptedInt": 44 + } + ] + }, + { + "object": "collection2", + "name": "find", + "arguments": { + "filter": {} + }, + "expectResult": [ + { + "_id": 1, + "encryptedInt": { + "$$type": "binData" + }, + "__safeContent__": { + "$$type": "array" + } + }, + { + "_id": 2, + "encryptedInt": { + "$$type": "binData" + }, + "__safeContent__": { + "$$type": "array" + } + }, + { + "_id": 3, + "encryptedInt": { + "$$type": "binData" + }, + "__safeContent__": { + "$$type": "array" + } + } + ] + } + ] + }, + { + "description": "client bulkWrite QE with multiple replace fails", + "operations": [ + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "models": [ + { + "replaceOne": { + "namespace": "crud-tests.coll0", + "filter": { + "encryptedInt": { + "$eq": 11 + } + }, + "replacement": { + "encryptedInt": 44 + } + } + }, + { + "replaceOne": { + "namespace": "crud-tests.coll0", + "filter": { + "encryptedInt": { + "$eq": 22 + } + }, + "replacement": { + "encryptedInt": 44 + } + } + } + ] + }, + "expectError": { + "isError": true, + "errorContains": "Only insert is supported in BulkWrite with multiple operations and Queryable Encryption" + } + } + ] + } + ] +} diff --git a/src/libmongoc/tests/test-mongoc-client-side-encryption.c b/src/libmongoc/tests/test-mongoc-client-side-encryption.c index 4bc9fb3a209..fc5d71377c8 100644 --- a/src/libmongoc/tests/test-mongoc-client-side-encryption.c +++ b/src/libmongoc/tests/test-mongoc-client-side-encryption.c @@ -334,6 +334,7 @@ _make_kms_masterkey(char const *provider) typedef struct { int num_inserts; + int num_bulk_writes; } limits_apm_ctx_t; static void @@ -342,14 +343,18 @@ _command_started(const mongoc_apm_command_started_t *event) limits_apm_ctx_t *ctx; ctx = (limits_apm_ctx_t *)mongoc_apm_command_started_get_context(event); - if (0 == strcmp("insert", mongoc_apm_command_started_get_command_name(event))) { + const char *cmd_name = mongoc_apm_command_started_get_command_name(event); + if (0 == strcmp("insert", cmd_name)) { ctx->num_inserts++; } + if (0 == strcmp("bulkWrite", cmd_name)) { + ctx->num_bulk_writes++; + } } /* Prose Test 4: BSON Size Limits and Batch Splitting */ static void -test_bson_size_limits_and_batch_splitting(void *unused) +test_bson_size_limits_and_batch_splitting(bool with_qe) { /* Expect an insert of two documents over 2MiB to split into two inserts but * still succeed. */ @@ -372,8 +377,6 @@ test_bson_size_limits_and_batch_splitting(void *unused) const int exceeds_2mib_after_encryption = size_2mib - 2000; const int exceeds_16mib_after_encryption = size_16mib - 2000; - BSON_UNUSED(unused); - /* Do the test setup. */ /* Drop and create db.coll configured with limits-schema.json */ @@ -499,6 +502,71 @@ test_bson_size_limits_and_batch_splitting(void *unused) } bson_destroy(docs[0]); + if (with_qe) { + mongoc_bulkwriteopts_t *bw_opts = mongoc_bulkwriteopts_new(); + mongoc_bulkwriteopts_set_verboseresults(bw_opts, true); + + bson_t *corpus_encryptedFields = + get_bson_from_json_file("./src/libmongoc/tests/client_side_encryption_prose/limits-encryptedFields.json"); + bson_t *coll_opts = BCON_NEW("encryptedFields", BCON_DOCUMENT(corpus_encryptedFields)); + mongoc_database_t *db = mongoc_client_get_database(client, "db"); + (void)mongoc_collection_drop(coll, NULL); + // Create a newly named collection to avoid cached previous JSON Schema. + mongoc_collection_t *coll2 = mongoc_database_create_collection(db, "coll2", coll_opts, &error); + ASSERT_OR_PRINT(coll2, error); + mongoc_collection_destroy(coll2); + + /* Insert two documents that each exceed 2MiB but no encryption occurs. + * Expect two separate bulkWrite commands. + */ + docs[0] = BCON_NEW("_id", "over_2mib_3"); + bson_append_utf8(docs[0], "unencrypted", -1, as, size_2mib - 1500); + docs[1] = BCON_NEW("_id", "over_2mib_4"); + bson_append_utf8(docs[1], "unencrypted", -1, as, size_2mib - 1500); + + mongoc_bulkwrite_t *bw = mongoc_client_bulkwrite_new(client); + ASSERT_OR_PRINT(mongoc_bulkwrite_append_insertone(bw, "db.coll2", docs[0], NULL, &error), error); + ASSERT_OR_PRINT(mongoc_bulkwrite_append_insertone(bw, "db.coll2", docs[1], NULL, &error), error); + + ctx.num_bulk_writes = 0; + mongoc_bulkwritereturn_t bwr = mongoc_bulkwrite_execute(bw, bw_opts); + ASSERT_NO_BULKWRITEEXCEPTION(bwr); + ASSERT_CMPINT(ctx.num_bulk_writes, ==, 2); + bson_destroy(docs[0]); + bson_destroy(docs[1]); + mongoc_bulkwrite_destroy(bw); + mongoc_bulkwriteresult_destroy(bwr.res); + mongoc_bulkwriteexception_destroy(bwr.exc); + + /* Insert two documents that each exceed 2MiB after encryption occurs. Expect + * the bulk write to succeed and run as two separate bulkWrite commands. + */ + docs[0] = get_bson_from_json_file("./src/libmongoc/tests/client_side_encryption_prose/limits-qe-doc.json"); + bson_append_utf8(docs[0], "_id", -1, "encryption_exceeds_2mib_3", -1); + bson_append_utf8(docs[0], "foo", -1, as, exceeds_2mib_after_encryption - 1500); + docs[1] = get_bson_from_json_file("./src/libmongoc/tests/client_side_encryption_prose/limits-qe-doc.json"); + bson_append_utf8(docs[1], "_id", -1, "encryption_exceeds_2mib_4", -1); + bson_append_utf8(docs[1], "foo", -1, as, exceeds_2mib_after_encryption - 1500); + + bw = mongoc_client_bulkwrite_new(client); + ASSERT_OR_PRINT(mongoc_bulkwrite_append_insertone(bw, "db.coll2", docs[0], NULL, &error), error); + ASSERT_OR_PRINT(mongoc_bulkwrite_append_insertone(bw, "db.coll2", docs[1], NULL, &error), error); + + ctx.num_bulk_writes = 0; + bwr = mongoc_bulkwrite_execute(bw, bw_opts); + ASSERT_NO_BULKWRITEEXCEPTION(bwr); + ASSERT_CMPINT(ctx.num_bulk_writes, ==, 2); + bson_destroy(docs[0]); + bson_destroy(docs[1]); + mongoc_bulkwrite_destroy(bw); + mongoc_bulkwriteresult_destroy(bwr.res); + mongoc_bulkwriteexception_destroy(bwr.exc); + mongoc_bulkwriteopts_destroy(bw_opts); + bson_destroy(corpus_encryptedFields); + bson_destroy(coll_opts); + mongoc_database_destroy(db); + } + bson_free(as); bson_destroy(kms_providers); bson_destroy(corpus_schema); @@ -511,6 +579,20 @@ test_bson_size_limits_and_batch_splitting(void *unused) mongoc_auto_encryption_opts_destroy(opts); } +static void +test_bson_size_limits_and_batch_splitting_no_qe(void *unused) +{ + BSON_UNUSED(unused); + test_bson_size_limits_and_batch_splitting(false); +} + +static void +test_bson_size_limits_and_batch_splitting_qe(void *unused) +{ + BSON_UNUSED(unused); + test_bson_size_limits_and_batch_splitting(true); +} + typedef struct { bson_t *last_cmd; } _datakey_and_double_encryption_ctx_t; @@ -6900,11 +6982,19 @@ test_client_side_encryption_install(TestSuite *suite) test_framework_skip_if_no_auth /* requires auth for error check */); TestSuite_AddFull(suite, "/client_side_encryption/bson_size_limits_and_batch_splitting", - test_bson_size_limits_and_batch_splitting, + test_bson_size_limits_and_batch_splitting_no_qe, NULL, NULL, test_framework_skip_if_no_client_side_encryption, TestSuite_CheckLive); + TestSuite_AddFull(suite, + "/client_side_encryption/bson_size_limits_and_batch_splitting_qe", + test_bson_size_limits_and_batch_splitting_qe, + NULL, + NULL, + test_framework_skip_if_no_client_side_encryption, + test_framework_skip_if_max_wire_version_less_than_25, + test_framework_skip_if_single); TestSuite_AddFull(suite, "/client_side_encryption/views_are_prohibited", test_views_are_prohibited, diff --git a/src/libmongoc/tests/test-mongoc-crud.c b/src/libmongoc/tests/test-mongoc-crud.c index de69ad66911..0671a4930e4 100644 --- a/src/libmongoc/tests/test-mongoc-crud.c +++ b/src/libmongoc/tests/test-mongoc-crud.c @@ -1259,57 +1259,6 @@ prose_test_12(void *ctx) mongoc_client_destroy(client); } -static void -prose_test_13(void *ctx) -{ - /* - 13. `MongoClient.bulkWrite` errors if configured with automatic encryption. - */ - mongoc_client_t *client; - BSON_UNUSED(ctx); - bool ok; - bson_error_t error; - - client = test_framework_new_default_client(); - mongoc_auto_encryption_opts_t *aeo = mongoc_auto_encryption_opts_new(); - mongoc_auto_encryption_opts_set_keyvault_namespace(aeo, "db", "coll"); - mongoc_auto_encryption_opts_set_kms_providers( - aeo, tmp_bson(BSON_STR({"aws" : {"accessKeyId" : "foo", "secretAccessKey" : "bar"}}))); - ok = mongoc_client_enable_auto_encryption(client, aeo, &error); - ASSERT_OR_PRINT(ok, error); - - // Try to to a bulk write. - { - mongoc_bulkwrite_t *bw = mongoc_client_bulkwrite_new(client); - - // Create bulk write. - { - ok = mongoc_bulkwrite_append_insertone(bw, "db.coll", tmp_bson("{'a': 'b'}"), NULL, &error); - ASSERT_OR_PRINT(ok, error); - } - - // Execute. - { - mongoc_bulkwritereturn_t bwr = mongoc_bulkwrite_execute(bw, NULL); - ASSERT(!bwr.res); // No result due to no successful writes. - ASSERT(bwr.exc); - if (!mongoc_bulkwriteexception_error(bwr.exc, &error)) { - test_error("Expected top-level error but got:\n%s", test_bulkwriteexception_str(bwr.exc)); - } - ASSERT_ERROR_CONTAINS(error, - MONGOC_ERROR_COMMAND, - MONGOC_ERROR_COMMAND_INVALID_ARG, - "bulkWrite does not currently support automatic encryption"); - mongoc_bulkwriteresult_destroy(bwr.res); - mongoc_bulkwriteexception_destroy(bwr.exc); - } - mongoc_bulkwrite_destroy(bw); - } - - mongoc_auto_encryption_opts_destroy(aeo); - mongoc_client_destroy(client); -} - static void prose_test_15(void *ctx) { @@ -1519,14 +1468,6 @@ test_crud_install(TestSuite *suite) test_framework_skip_if_max_wire_version_less_than_25 // require server 8.0 ); - TestSuite_AddFull(suite, - "/crud/prose_test_13", - prose_test_13, - NULL /* dtor */, - NULL /* ctx */, - test_framework_skip_if_max_wire_version_less_than_25, // require server 8.0 - test_framework_skip_if_no_client_side_encryption); - TestSuite_AddFull(suite, "/crud/prose_test_15", prose_test_15,