From c6f645d2ba290b8f1aac6bfad8215f150da06932 Mon Sep 17 00:00:00 2001 From: mdb-ad Date: Wed, 22 Jan 2025 16:08:27 -0800 Subject: [PATCH 01/33] add qe to unified tests --- .../client-side-encryption.md | 2 +- source/unified-test-format/schema-1.23.json | 1133 +++++++++++++++++ .../valid-pass/poc-queryable-encryption.json | 179 +++ .../valid-pass/poc-queryable-encryption.yml | 80 ++ .../unified-test-format.md | 29 +- 5 files changed, 1408 insertions(+), 15 deletions(-) create mode 100644 source/unified-test-format/schema-1.23.json create mode 100644 source/unified-test-format/tests/valid-pass/poc-queryable-encryption.json create mode 100644 source/unified-test-format/tests/valid-pass/poc-queryable-encryption.yml diff --git a/source/client-side-encryption/client-side-encryption.md b/source/client-side-encryption/client-side-encryption.md index 2eee97e0c9..1576dea211 100644 --- a/source/client-side-encryption/client-side-encryption.md +++ b/source/client-side-encryption/client-side-encryption.md @@ -159,7 +159,7 @@ supports indexed encrypted fields, which are further processed server-side. Is an umbrella term describing the both CSFLE and Queryable Encryption. -**encryptedFields** + **encryptedFields** A BSON document describing the Queryable Encryption encrypted fields. This is analogous to the JSON Schema in FLE. The following is an example encryptedFields in extended canonical JSON: diff --git a/source/unified-test-format/schema-1.23.json b/source/unified-test-format/schema-1.23.json new file mode 100644 index 0000000000..dc716c7fa5 --- /dev/null +++ b/source/unified-test-format/schema-1.23.json @@ -0,0 +1,1133 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Unified Test Format", + "type": "object", + "additionalProperties": false, + "required": [ + "description", + "schemaVersion", + "tests" + ], + "properties": { + "description": { + "type": "string" + }, + "schemaVersion": { + "$ref": "#/definitions/version" + }, + "runOnRequirements": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/runOnRequirement" + } + }, + "createEntities": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/entity" + } + }, + "initialData": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/collectionData" + } + }, + "tests": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/test" + } + }, + "_yamlAnchors": { + "type": "object", + "additionalProperties": true + } + }, + "definitions": { + "version": { + "type": "string", + "pattern": "^[0-9]+(\\.[0-9]+){1,2}$" + }, + "runOnRequirement": { + "type": "object", + "additionalProperties": false, + "minProperties": 1, + "properties": { + "maxServerVersion": { + "$ref": "#/definitions/version" + }, + "minServerVersion": { + "$ref": "#/definitions/version" + }, + "topologies": { + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "enum": [ + "single", + "replicaset", + "sharded", + "sharded-replicaset", + "load-balanced" + ] + } + }, + "serverless": { + "type": "string", + "enum": [ + "require", + "forbid", + "allow" + ] + }, + "serverParameters": { + "type": "object", + "minProperties": 1 + }, + "auth": { + "type": "boolean" + }, + "authMechanism": { + "type": "string" + }, + "csfle": { + "type": "boolean" + } + } + }, + "entity": { + "type": "object", + "additionalProperties": false, + "maxProperties": 1, + "minProperties": 1, + "properties": { + "client": { + "type": "object", + "additionalProperties": false, + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string" + }, + "uriOptions": { + "type": "object" + }, + "useMultipleMongoses": { + "type": "boolean" + }, + "observeEvents": { + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "enum": [ + "commandStartedEvent", + "commandSucceededEvent", + "commandFailedEvent", + "poolCreatedEvent", + "poolReadyEvent", + "poolClearedEvent", + "poolClosedEvent", + "connectionCreatedEvent", + "connectionReadyEvent", + "connectionClosedEvent", + "connectionCheckOutStartedEvent", + "connectionCheckOutFailedEvent", + "connectionCheckedOutEvent", + "connectionCheckedInEvent", + "serverDescriptionChangedEvent", + "topologyDescriptionChangedEvent" + ] + } + }, + "ignoreCommandMonitoringEvents": { + "type": "array", + "minItems": 1, + "items": { + "type": "string" + } + }, + "storeEventsAsEntities": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/storeEventsAsEntity" + } + }, + "observeLogMessages": { + "type": "object", + "minProperties": 1, + "additionalProperties": false, + "properties": { + "command": { + "$ref": "#/definitions/logSeverityLevel" + }, + "topology": { + "$ref": "#/definitions/logSeverityLevel" + }, + "serverSelection": { + "$ref": "#/definitions/logSeverityLevel" + }, + "connection": { + "$ref": "#/definitions/logSeverityLevel" + } + } + }, + "serverApi": { + "$ref": "#/definitions/serverApi" + }, + "observeSensitiveCommands": { + "type": "boolean" + }, + "autoEncryptOpts": { + "keyVaultNamespace": { + "type": "string" + }, + "bypassAutoEncryption": { + "type": "boolean" + }, + "kmsProviders": { + "$ref": "#/definitions/kmsProviders" + } + } + } + }, + "clientEncryption": { + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "clientEncryptionOpts" + ], + "properties": { + "id": { + "type": "string" + }, + "clientEncryptionOpts": { + "$ref": "#/definitions/clientEncryptionOpts" + } + } + }, + "database": { + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "client", + "databaseName" + ], + "properties": { + "id": { + "type": "string" + }, + "client": { + "type": "string" + }, + "databaseName": { + "type": "string" + }, + "databaseOptions": { + "$ref": "#/definitions/collectionOrDatabaseOptions" + } + } + }, + "collection": { + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "database", + "collectionName" + ], + "properties": { + "id": { + "type": "string" + }, + "database": { + "type": "string" + }, + "collectionName": { + "type": "string" + }, + "collectionOptions": { + "$ref": "#/definitions/collectionOrDatabaseOptions" + } + } + }, + "session": { + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "client" + ], + "properties": { + "id": { + "type": "string" + }, + "client": { + "type": "string" + }, + "sessionOptions": { + "type": "object" + } + } + }, + "bucket": { + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "database" + ], + "properties": { + "id": { + "type": "string" + }, + "database": { + "type": "string" + }, + "bucketOptions": { + "type": "object" + } + } + }, + "thread": { + "type": "object", + "additionalProperties": false, + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string" + } + } + } + } + }, + "logComponent": { + "type": "string", + "enum": [ + "command", + "topology", + "serverSelection", + "connection" + ] + }, + "logSeverityLevel": { + "type": "string", + "enum": [ + "emergency", + "alert", + "critical", + "error", + "warning", + "notice", + "info", + "debug", + "trace" + ] + }, + "clientEncryptionOpts": { + "type": "object", + "additionalProperties": false, + "required": [ + "keyVaultClient", + "keyVaultNamespace", + "kmsProviders" + ], + "properties": { + "keyVaultClient": { + "type": "string" + }, + "keyVaultNamespace": { + "type": "string" + }, + "kmsProviders": { + "$ref": "#/definitions/kmsProviders" + }, + "keyExpirationMS": { + "type": "integer" + } + } + }, + "kmsProviders": { + "$defs": { + "stringOrPlaceholder": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "$$placeholder" + ], + "properties": { + "$$placeholder": {} + } + } + ] + } + }, + "type": "object", + "additionalProperties": false, + "patternProperties": { + "^aws(:[a-zA-Z0-9_]+)?$": { + "type": "object", + "additionalProperties": false, + "properties": { + "accessKeyId": { + "$ref": "#/definitions/kmsProviders/$defs/stringOrPlaceholder" + }, + "secretAccessKey": { + "$ref": "#/definitions/kmsProviders/$defs/stringOrPlaceholder" + }, + "sessionToken": { + "$ref": "#/definitions/kmsProviders/$defs/stringOrPlaceholder" + } + } + }, + "^azure(:[a-zA-Z0-9_]+)?$": { + "type": "object", + "additionalProperties": false, + "properties": { + "tenantId": { + "$ref": "#/definitions/kmsProviders/$defs/stringOrPlaceholder" + }, + "clientId": { + "$ref": "#/definitions/kmsProviders/$defs/stringOrPlaceholder" + }, + "clientSecret": { + "$ref": "#/definitions/kmsProviders/$defs/stringOrPlaceholder" + }, + "identityPlatformEndpoint": { + "$ref": "#/definitions/kmsProviders/$defs/stringOrPlaceholder" + } + } + }, + "^gcp(:[a-zA-Z0-9_]+)?$": { + "type": "object", + "additionalProperties": false, + "properties": { + "email": { + "$ref": "#/definitions/kmsProviders/$defs/stringOrPlaceholder" + }, + "privateKey": { + "$ref": "#/definitions/kmsProviders/$defs/stringOrPlaceholder" + }, + "endpoint": { + "$ref": "#/definitions/kmsProviders/$defs/stringOrPlaceholder" + } + } + }, + "^kmip(:[a-zA-Z0-9_]+)?$": { + "type": "object", + "additionalProperties": false, + "properties": { + "endpoint": { + "$ref": "#/definitions/kmsProviders/$defs/stringOrPlaceholder" + } + } + }, + "^local(:[a-zA-Z0-9_]+)?$": { + "type": "object", + "additionalProperties": false, + "properties": { + "key": { + "$ref": "#/definitions/kmsProviders/$defs/stringOrPlaceholder" + } + } + } + } + }, + "storeEventsAsEntity": { + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "events" + ], + "properties": { + "id": { + "type": "string" + }, + "events": { + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "enum": [ + "PoolCreatedEvent", + "PoolReadyEvent", + "PoolClearedEvent", + "PoolClosedEvent", + "ConnectionCreatedEvent", + "ConnectionReadyEvent", + "ConnectionClosedEvent", + "ConnectionCheckOutStartedEvent", + "ConnectionCheckOutFailedEvent", + "ConnectionCheckedOutEvent", + "ConnectionCheckedInEvent", + "CommandStartedEvent", + "CommandSucceededEvent", + "CommandFailedEvent", + "ServerDescriptionChangedEvent", + "TopologyDescriptionChangedEvent" + ] + } + } + } + }, + "collectionData": { + "type": "object", + "additionalProperties": false, + "required": [ + "collectionName", + "databaseName", + "documents" + ], + "properties": { + "collectionName": { + "type": "string" + }, + "databaseName": { + "type": "string" + }, + "createOptions": { + "type": "object", + "properties": { + "writeConcern": false + } + }, + "documents": { + "type": "array", + "items": { + "type": "object" + } + } + } + }, + "expectedEventsForClient": { + "type": "object", + "additionalProperties": false, + "required": [ + "client", + "events" + ], + "properties": { + "client": { + "type": "string" + }, + "eventType": { + "type": "string", + "enum": [ + "command", + "cmap", + "sdam" + ] + }, + "events": { + "type": "array" + }, + "ignoreExtraEvents": { + "type": "boolean" + } + }, + "oneOf": [ + { + "required": [ + "eventType" + ], + "properties": { + "eventType": { + "const": "command" + }, + "events": { + "type": "array", + "items": { + "$ref": "#/definitions/expectedCommandEvent" + } + } + } + }, + { + "required": [ + "eventType" + ], + "properties": { + "eventType": { + "const": "cmap" + }, + "events": { + "type": "array", + "items": { + "$ref": "#/definitions/expectedCmapEvent" + } + } + } + }, + { + "required": [ + "eventType" + ], + "properties": { + "eventType": { + "const": "sdam" + }, + "events": { + "type": "array", + "items": { + "$ref": "#/definitions/expectedSdamEvent" + } + } + } + }, + { + "additionalProperties": false, + "properties": { + "client": { + "type": "string" + }, + "events": { + "type": "array", + "items": { + "$ref": "#/definitions/expectedCommandEvent" + } + }, + "ignoreExtraEvents": { + "type": "boolean" + } + } + } + ] + }, + "expectedCommandEvent": { + "type": "object", + "additionalProperties": false, + "maxProperties": 1, + "minProperties": 1, + "properties": { + "commandStartedEvent": { + "type": "object", + "additionalProperties": false, + "properties": { + "command": { + "type": "object" + }, + "commandName": { + "type": "string" + }, + "databaseName": { + "type": "string" + }, + "hasServiceId": { + "type": "boolean" + }, + "hasServerConnectionId": { + "type": "boolean" + } + } + }, + "commandSucceededEvent": { + "type": "object", + "additionalProperties": false, + "properties": { + "reply": { + "type": "object" + }, + "commandName": { + "type": "string" + }, + "databaseName": { + "type": "string" + }, + "hasServiceId": { + "type": "boolean" + }, + "hasServerConnectionId": { + "type": "boolean" + } + } + }, + "commandFailedEvent": { + "type": "object", + "additionalProperties": false, + "properties": { + "commandName": { + "type": "string" + }, + "databaseName": { + "type": "string" + }, + "hasServiceId": { + "type": "boolean" + }, + "hasServerConnectionId": { + "type": "boolean" + } + } + } + } + }, + "expectedCmapEvent": { + "type": "object", + "additionalProperties": false, + "maxProperties": 1, + "minProperties": 1, + "properties": { + "poolCreatedEvent": { + "type": "object", + "additionalProperties": false, + "properties": {} + }, + "poolReadyEvent": { + "type": "object", + "additionalProperties": false, + "properties": {} + }, + "poolClearedEvent": { + "type": "object", + "additionalProperties": false, + "properties": { + "hasServiceId": { + "type": "boolean" + }, + "interruptInUseConnections": { + "type": "boolean" + } + } + }, + "poolClosedEvent": { + "type": "object", + "additionalProperties": false, + "properties": {} + }, + "connectionCreatedEvent": { + "type": "object", + "additionalProperties": false, + "properties": {} + }, + "connectionReadyEvent": { + "type": "object", + "additionalProperties": false, + "properties": {} + }, + "connectionClosedEvent": { + "type": "object", + "additionalProperties": false, + "properties": { + "reason": { + "type": "string" + } + } + }, + "connectionCheckOutStartedEvent": { + "type": "object", + "additionalProperties": false, + "properties": {} + }, + "connectionCheckOutFailedEvent": { + "type": "object", + "additionalProperties": false, + "properties": { + "reason": { + "type": "string" + } + } + }, + "connectionCheckedOutEvent": { + "type": "object", + "additionalProperties": false, + "properties": {} + }, + "connectionCheckedInEvent": { + "type": "object", + "additionalProperties": false, + "properties": {} + } + } + }, + "expectedSdamEvent": { + "type": "object", + "additionalProperties": false, + "maxProperties": 1, + "minProperties": 1, + "properties": { + "serverDescriptionChangedEvent": { + "type": "object", + "additionalProperties": false, + "properties": { + "previousDescription": { + "$ref": "#/definitions/serverDescription" + }, + "newDescription": { + "$ref": "#/definitions/serverDescription" + } + } + }, + "topologyDescriptionChangedEvent": { + "type": "object", + "additionalProperties": false, + "properties": { + "previousDescription": { + "$ref": "#/definitions/topologyDescription" + }, + "newDescription": { + "$ref": "#/definitions/topologyDescription" + } + } + }, + "serverHeartbeatStartedEvent": { + "type": "object", + "additionalProperties": false, + "properties": { + "awaited": { + "type": "boolean" + } + } + }, + "serverHeartbeatSucceededEvent": { + "type": "object", + "additionalProperties": false, + "properties": { + "awaited": { + "type": "boolean" + } + } + }, + "serverHeartbeatFailedEvent": { + "type": "object", + "additionalProperties": false, + "properties": { + "awaited": { + "type": "boolean" + } + } + }, + "topologyOpeningEvent": { + "type": "object", + "additionalProperties": false, + "properties": {} + }, + "topologyClosedEvent": { + "type": "object", + "additionalProperties": false, + "properties": {} + } + } + }, + "serverDescription": { + "type": "object", + "additionalProperties": false, + "properties": { + "type": { + "type": "string", + "enum": [ + "Standalone", + "Mongos", + "PossiblePrimary", + "RSPrimary", + "RSSecondary", + "RSOther", + "RSArbiter", + "RSGhost", + "LoadBalancer", + "Unknown" + ] + } + } + }, + "topologyDescription": { + "type": "object", + "additionalProperties": false, + "properties": { + "type": { + "type": "string", + "enum": [ + "Single", + "Unknown", + "ReplicaSetNoPrimary", + "ReplicaSetWithPrimary", + "Sharded", + "LoadBalanced" + ] + } + } + }, + "expectedLogMessagesForClient": { + "type": "object", + "additionalProperties": false, + "required": [ + "client", + "messages" + ], + "properties": { + "client": { + "type": "string" + }, + "messages": { + "type": "array", + "items": { + "$ref": "#/definitions/expectedLogMessage" + } + }, + "ignoreExtraMessages": { + "type": "boolean" + }, + "ignoreMessages": { + "type": "array", + "items": { + "$ref": "#/definitions/expectedLogMessage" + } + } + } + }, + "expectedLogMessage": { + "type": "object", + "additionalProperties": false, + "required": [ + "level", + "component", + "data" + ], + "properties": { + "level": { + "$ref": "#/definitions/logSeverityLevel" + }, + "component": { + "$ref": "#/definitions/logComponent" + }, + "data": { + "type": "object" + }, + "failureIsRedacted": { + "type": "boolean" + } + } + }, + "collectionOrDatabaseOptions": { + "type": "object", + "additionalProperties": false, + "properties": { + "readConcern": { + "type": "object" + }, + "readPreference": { + "type": "object" + }, + "writeConcern": { + "type": "object" + }, + "timeoutMS": { + "type": "integer" + }, + "encryptedFields": { + "type": "object" + } + } + }, + "serverApi": { + "type": "object", + "additionalProperties": false, + "required": [ + "version" + ], + "properties": { + "version": { + "type": "string" + }, + "strict": { + "type": "boolean" + }, + "deprecationErrors": { + "type": "boolean" + } + } + }, + "operation": { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "object" + ], + "properties": { + "name": { + "type": "string" + }, + "object": { + "type": "string" + }, + "arguments": { + "type": "object" + }, + "ignoreResultAndError": { + "type": "boolean" + }, + "expectError": { + "$ref": "#/definitions/expectedError" + }, + "expectResult": {}, + "saveResultAsEntity": { + "type": "string" + } + }, + "allOf": [ + { + "not": { + "required": [ + "expectError", + "expectResult" + ] + } + }, + { + "not": { + "required": [ + "expectError", + "saveResultAsEntity" + ] + } + }, + { + "not": { + "required": [ + "ignoreResultAndError", + "expectResult" + ] + } + }, + { + "not": { + "required": [ + "ignoreResultAndError", + "expectError" + ] + } + }, + { + "not": { + "required": [ + "ignoreResultAndError", + "saveResultAsEntity" + ] + } + } + ] + }, + "expectedError": { + "type": "object", + "additionalProperties": false, + "minProperties": 1, + "properties": { + "isError": { + "type": "boolean", + "const": true + }, + "isClientError": { + "type": "boolean" + }, + "isTimeoutError": { + "type": "boolean" + }, + "errorContains": { + "type": "string" + }, + "errorCode": { + "type": "integer" + }, + "errorCodeName": { + "type": "string" + }, + "errorLabelsContain": { + "type": "array", + "minItems": 1, + "items": { + "type": "string" + } + }, + "errorLabelsOmit": { + "type": "array", + "minItems": 1, + "items": { + "type": "string" + } + }, + "writeErrors": { + "type": "object" + }, + "writeConcernErrors": { + "type": "array", + "items": { + "type": "object" + } + }, + "errorResponse": { + "type": "object" + }, + "expectResult": {} + } + }, + "test": { + "type": "object", + "additionalProperties": false, + "required": [ + "description", + "operations" + ], + "properties": { + "description": { + "type": "string" + }, + "runOnRequirements": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/runOnRequirement" + } + }, + "skipReason": { + "type": "string" + }, + "operations": { + "type": "array", + "items": { + "$ref": "#/definitions/operation" + } + }, + "expectEvents": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/expectedEventsForClient" + } + }, + "expectLogMessages": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/expectedLogMessagesForClient" + } + }, + "outcome": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/collectionData" + } + } + } + } + } +} \ No newline at end of file diff --git a/source/unified-test-format/tests/valid-pass/poc-queryable-encryption.json b/source/unified-test-format/tests/valid-pass/poc-queryable-encryption.json new file mode 100644 index 0000000000..c43d22631f --- /dev/null +++ b/source/unified-test-format/tests/valid-pass/poc-queryable-encryption.json @@ -0,0 +1,179 @@ +{ + "description": "client bulkWrite updateOne-sort", + "schemaVersion": "1.23", + "runOnRequirements": [ + { + "minServerVersion": "8.0" + } + ], + "createEntities": [ + { + "client": { + "id": "client0", + "observeEvents": [ + "commandStartedEvent", + "commandSucceededEvent" + ], + "autoEncryptOpts": { + "kmsProviders": { + "local": { + "key": { + "$binary": { + "base64": "Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk", + "subType": "00" + } + } + } + } + } + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "crud-tests" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "coll0", + "collectionOptions": { + "encryptedFields": { + "fields": [ + { + "keyId": { + "$binary": { + "base64": "EjRWeBI0mHYSNBI0VniQEg==", + "subType": "04" + } + }, + "path": "encryptedInt", + "bsonType": "int", + "queries": { + "queryType": "equality", + "contention": { + "$numberLong": "0" + } + } + } + ] + } + } + } + }, + { + "database": { + "id": "database1", + "client": "client0", + "databaseName": "keyvault" + } + }, + { + "collection": { + "id": "collection1", + "database": "database0", + "collectionName": "datakeys" + } + } + ], + "initialData": [ + { + "databaseName": "keyvault", + "collectionName": "datakeys", + "documents": [ + { + "_id": { + "$binary": { + "base64": "EjRWeBI0mHYSNBI0VniQEg==", + "subType": "04" + } + }, + "keyMaterial": { + "$binary": { + "base64": "sHe0kz57YW7v8g9VP9sf/+K1ex4JqKc5rf/URX3n3p8XdZ6+15uXPaSayC6adWbNxkFskuMCOifDoTT+rkqMtFkDclOy884RuGGtUysq3X7zkAWYTKi8QAfKkajvVbZl2y23UqgVasdQu3OVBQCrH/xY00nNAs/52e958nVjBuzQkSb1T8pKJAyjZsHJ60+FtnfafDZSTAIBJYn7UWBCwQ==", + "subType": "00" + } + }, + "creationDate": { + "$date": { + "$numberLong": "1641024000000" + } + }, + "updateDate": { + "$date": { + "$numberLong": "1641024000000" + } + }, + "status": 1, + "masterKey": { + "provider": "local" + } + } + ] + } + ], + "tests": [ + { + "description": "insert, replace, and find with queryable encryption", + "operations": [ + { + "object": "collection0", + "name": "insertOne", + "arguments": { + "document": { + "_id": 1, + "encryptedInt": 11 + } + } + }, + { + "object": "collection0", + "name": "replaceOne", + "arguments": { + "filter": { + "encryptedInt": 11 + }, + "replacement": { + "encryptedInt": 22 + } + } + }, + { + "object": "collection0", + "name": "find", + "arguments": { + "filter": { + "encryptedInt": 22 + } + }, + "expectResult": [ + { + "_id": 1, + "encryptedInt": 22 + } + ] + } + ], + "outcome": [ + { + "collectionName": "coll0", + "databaseName": "crud-tests", + "documents": [ + { + "_id": 1, + "encryptedInt": { + "$$type": "binData" + }, + "__safeContent__": { + "$$type": "array" + } + } + ] + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/valid-pass/poc-queryable-encryption.yml b/source/unified-test-format/tests/valid-pass/poc-queryable-encryption.yml new file mode 100644 index 0000000000..d5604cd1ea --- /dev/null +++ b/source/unified-test-format/tests/valid-pass/poc-queryable-encryption.yml @@ -0,0 +1,80 @@ +description: client bulkWrite updateOne-sort + +schemaVersion: "1.23" + +runOnRequirements: + - minServerVersion: "8.0" + +createEntities: + - client: + id: &client0 client0 + observeEvents: + - commandStartedEvent + - commandSucceededEvent + autoEncryptOpts: + kmsProviders: + local: {'key': {'$binary': {'base64': 'Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk', 'subType': '00'}}} + - database: + id: &database0 database0 + client: *client0 + databaseName: &database0Name crud-tests + - collection: + id: &collection0 collection0 + database: *database0 + collectionName: &collection0Name coll0 + collectionOptions: + encryptedFields: + fields: + - keyId: + $binary: {'base64': 'EjRWeBI0mHYSNBI0VniQEg==', 'subType': '04'} + path: 'encryptedInt' + bsonType: 'int' + queries: {'queryType': 'equality', 'contention': {'$numberLong': '0'}} + - database: + id: &database1 database1 + client: *client0 + databaseName: &database1Name keyvault + - collection: + id: &collection1 collection1 + database: *database0 + collectionName: &collection1Name datakeys + + +initialData: + - databaseName: *database1Name + collectionName: *collection1Name + documents: + - _id: { $binary: { base64: EjRWeBI0mHYSNBI0VniQEg==, subType: "04" } } + keyMaterial: { $binary: { base64: sHe0kz57YW7v8g9VP9sf/+K1ex4JqKc5rf/URX3n3p8XdZ6+15uXPaSayC6adWbNxkFskuMCOifDoTT+rkqMtFkDclOy884RuGGtUysq3X7zkAWYTKi8QAfKkajvVbZl2y23UqgVasdQu3OVBQCrH/xY00nNAs/52e958nVjBuzQkSb1T8pKJAyjZsHJ60+FtnfafDZSTAIBJYn7UWBCwQ==, subType: "00" } } + creationDate: { $date: { $numberLong: "1641024000000" } } + updateDate: { $date: { $numberLong: "1641024000000" } } + status: 1 + masterKey: + provider: local + +tests: + - description: insert, replace, and find with queryable encryption + operations: + - object: *collection0 + name: insertOne + arguments: + document: + _id: 1 + encryptedInt: 11 + - object: *collection0 + name: replaceOne + arguments: + filter: { encryptedInt: 11 } + replacement: { encryptedInt: 22 } + - object: *collection0 + name: find + arguments: + filter: { encryptedInt: 22 } + expectResult: + - _id: 1 + encryptedInt: 22 + outcome: + - collectionName: *collection0Name + databaseName: *database0Name + documents: + - { _id: 1, encryptedInt: { $$type: binData }, __safeContent__: { $$type: array} } \ No newline at end of file diff --git a/source/unified-test-format/unified-test-format.md b/source/unified-test-format/unified-test-format.md index 6e56e63544..921ccfbe24 100644 --- a/source/unified-test-format/unified-test-format.md +++ b/source/unified-test-format/unified-test-format.md @@ -534,6 +534,14 @@ The structure of this object is as follows: client. - `serverApi`: Optional [serverApi](#serverapi) object. + + + - `autoEncryptOpts`: Optional object with the following fields: + - `kmsProviders`: The same as in [`clientEncryption`](#entity_clientEncryption). + - `keyVaultNamespace`: Optional, a namespace to the key vault collection. Defaults to "keyvault.datakeys". + - `bypassAutoEncryption`: Optional, a boolean to indicate whether or not auto encryption should be bypassed. + Defaults to `false`. + - `clientEncryption`: Optional object. Defines a ClientEncryption object. @@ -1307,6 +1315,8 @@ The structure of this object is as follows: - `readConcern`: Optional object. See [commonOptions_readConcern](#commonOptions_readConcern). - `readPreference`: Optional object. See [commonOptions_readPreference](#commonOptions_readPreference). - `writeConcern`: Optional object. See [commonOptions_writeConcern](#commonOptions_writeConcern). +- `encryptedFields`: Optional object. See + [Client Side Encryption: `encryptedFields`](../client-side-encryption/client-side-encryption.md#encryptedFields) ### Common Options @@ -2780,7 +2790,7 @@ Contexts where one might encounter a root-level document include: include: - [aggregate](#aggregate) - - [find](#find)) + - [find](#find) - [listCollections](#listcollections), listDatabases, and listIndexes - [listSearchIndexes](#listsearchindexes) - [runCursorCommand](#runcursorcommand) @@ -3522,19 +3532,6 @@ would need to represent streams as entities and support IO operations to directl entity. This may not be worth the added complexity if the existing operations provide adequate test coverage for GridFS implementations. -### Support Client-side Encryption integration tests - -Supporting client-side encryption spec tests will require the following changes to the test format: - -- `json_schema` will need to be specified when creating a collection, via either the collection entity definition or - [initialData](#initialData). -- `key_vault_data` can be expressed via [initialData](#initialData) -- `autoEncryptOpts` will need to be specified when defining a client entity. Preparation of this field may require - reading AWS credentials from environment variables. - -The process for executing tests should not require significant changes, but test files will need to express a dependency -on mongocryptd. - ### Incorporate referenced entity operations into the schema version The [Schema Version](#schema-version) is not impacted by changes to operations defined in other specs and referenced in @@ -3552,6 +3549,10 @@ other specs *and* collating spec changes developed in parallel or during the sam ## Changelog +- 2025-01-21: **Schema version 1.23.** + + Support queryable encryption. + - 2024-11-12: **Schema version 1.22.** Add `keyExpirationMS` to `clientEncryption` entity. From aa51fc3af532d7c8bba4b97584d449b745f7ba4a Mon Sep 17 00:00:00 2001 From: mdb-ad Date: Wed, 5 Feb 2025 20:40:24 -0800 Subject: [PATCH 02/33] move encryptedFields to initialData --- source/unified-test-format/schema-1.23.json | 25 ++++---- source/unified-test-format/tests/Makefile | 2 +- .../valid-pass/poc-queryable-encryption.json | 64 +++++++++---------- .../valid-pass/poc-queryable-encryption.yml | 26 ++++---- .../unified-test-format.md | 4 +- 5 files changed, 63 insertions(+), 58 deletions(-) diff --git a/source/unified-test-format/schema-1.23.json b/source/unified-test-format/schema-1.23.json index dc716c7fa5..6dd34a6cda 100644 --- a/source/unified-test-format/schema-1.23.json +++ b/source/unified-test-format/schema-1.23.json @@ -188,14 +188,20 @@ "type": "boolean" }, "autoEncryptOpts": { - "keyVaultNamespace": { - "type": "string" - }, - "bypassAutoEncryption": { - "type": "boolean" - }, - "kmsProviders": { - "$ref": "#/definitions/kmsProviders" + "type": "object", + "required": [ + "kmsProviders" + ], + "properties": { + "keyVaultNamespace": { + "type": "string" + }, + "bypassAutoEncryption": { + "type": "boolean" + }, + "kmsProviders": { + "$ref": "#/definitions/kmsProviders" + } } } } @@ -931,9 +937,6 @@ }, "timeoutMS": { "type": "integer" - }, - "encryptedFields": { - "type": "object" } } }, diff --git a/source/unified-test-format/tests/Makefile b/source/unified-test-format/tests/Makefile index b7b58cd7a6..1a049e72ce 100644 --- a/source/unified-test-format/tests/Makefile +++ b/source/unified-test-format/tests/Makefile @@ -1,4 +1,4 @@ -SCHEMA=../schema-1.22.json +SCHEMA=../schema-1.23.json .PHONY: all invalid valid-fail valid-pass atlas-data-lake versioned-api load-balancers gridfs transactions transactions-convenient-api crud collection-management read-write-concern retryable-reads retryable-writes sessions command-logging-and-monitoring client-side-operations-timeout HAS_AJV diff --git a/source/unified-test-format/tests/valid-pass/poc-queryable-encryption.json b/source/unified-test-format/tests/valid-pass/poc-queryable-encryption.json index c43d22631f..4fa5a350e0 100644 --- a/source/unified-test-format/tests/valid-pass/poc-queryable-encryption.json +++ b/source/unified-test-format/tests/valid-pass/poc-queryable-encryption.json @@ -1,5 +1,5 @@ { - "description": "client bulkWrite updateOne-sort", + "description": "poc-queryable-encryption", "schemaVersion": "1.23", "runOnRequirements": [ { @@ -17,12 +17,7 @@ "autoEncryptOpts": { "kmsProviders": { "local": { - "key": { - "$binary": { - "base64": "Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk", - "subType": "00" - } - } + "key": "Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk" } } } @@ -32,36 +27,14 @@ "database": { "id": "database0", "client": "client0", - "databaseName": "crud-tests" + "databaseName": "poc-queryable-encryption" } }, { "collection": { "id": "collection0", "database": "database0", - "collectionName": "coll0", - "collectionOptions": { - "encryptedFields": { - "fields": [ - { - "keyId": { - "$binary": { - "base64": "EjRWeBI0mHYSNBI0VniQEg==", - "subType": "04" - } - }, - "path": "encryptedInt", - "bsonType": "int", - "queries": { - "queryType": "equality", - "contention": { - "$numberLong": "0" - } - } - } - ] - } - } + "collectionName": "coll0" } }, { @@ -113,6 +86,33 @@ } } ] + }, + { + "databaseName": "poc-queryable-encryption", + "collectionName": "coll0", + "documents": [], + "createOptions": { + "encryptedFields": { + "fields": [ + { + "keyId": { + "$binary": { + "base64": "EjRWeBI0mHYSNBI0VniQEg==", + "subType": "04" + } + }, + "path": "encryptedInt", + "bsonType": "int", + "queries": { + "queryType": "equality", + "contention": { + "$numberLong": "0" + } + } + } + ] + } + } } ], "tests": [ @@ -160,7 +160,7 @@ "outcome": [ { "collectionName": "coll0", - "databaseName": "crud-tests", + "databaseName": "poc-queryable-encryption", "documents": [ { "_id": 1, diff --git a/source/unified-test-format/tests/valid-pass/poc-queryable-encryption.yml b/source/unified-test-format/tests/valid-pass/poc-queryable-encryption.yml index d5604cd1ea..9a81a40ed5 100644 --- a/source/unified-test-format/tests/valid-pass/poc-queryable-encryption.yml +++ b/source/unified-test-format/tests/valid-pass/poc-queryable-encryption.yml @@ -1,4 +1,4 @@ -description: client bulkWrite updateOne-sort +description: poc-queryable-encryption schemaVersion: "1.23" @@ -13,23 +13,16 @@ createEntities: - commandSucceededEvent autoEncryptOpts: kmsProviders: - local: {'key': {'$binary': {'base64': 'Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk', 'subType': '00'}}} + local: + key: Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk - database: id: &database0 database0 client: *client0 - databaseName: &database0Name crud-tests + databaseName: &database0Name poc-queryable-encryption - collection: id: &collection0 collection0 database: *database0 collectionName: &collection0Name coll0 - collectionOptions: - encryptedFields: - fields: - - keyId: - $binary: {'base64': 'EjRWeBI0mHYSNBI0VniQEg==', 'subType': '04'} - path: 'encryptedInt' - bsonType: 'int' - queries: {'queryType': 'equality', 'contention': {'$numberLong': '0'}} - database: id: &database1 database1 client: *client0 @@ -51,6 +44,17 @@ initialData: status: 1 masterKey: provider: local + - databaseName: *database0Name + collectionName: *collection0Name + documents: [] + createOptions: + encryptedFields: + fields: + - keyId: + $binary: {'base64': 'EjRWeBI0mHYSNBI0VniQEg==', 'subType': '04'} + path: 'encryptedInt' + bsonType: 'int' + queries: {'queryType': 'equality', 'contention': {'$numberLong': '0'}} tests: - description: insert, replace, and find with queryable encryption diff --git a/source/unified-test-format/unified-test-format.md b/source/unified-test-format/unified-test-format.md index 921ccfbe24..193e6208b0 100644 --- a/source/unified-test-format/unified-test-format.md +++ b/source/unified-test-format/unified-test-format.md @@ -1315,8 +1315,6 @@ The structure of this object is as follows: - `readConcern`: Optional object. See [commonOptions_readConcern](#commonOptions_readConcern). - `readPreference`: Optional object. See [commonOptions_readPreference](#commonOptions_readPreference). - `writeConcern`: Optional object. See [commonOptions_writeConcern](#commonOptions_writeConcern). -- `encryptedFields`: Optional object. See - [Client Side Encryption: `encryptedFields`](../client-side-encryption/client-side-encryption.md#encryptedFields) ### Common Options @@ -3551,7 +3549,7 @@ other specs *and* collating spec changes developed in parallel or during the sam - 2025-01-21: **Schema version 1.23.** - Support queryable encryption. + Support automatic encryption. Add `autoEncryptOpts` to `client` entity. - 2024-11-12: **Schema version 1.22.** From fe8ed1ca43374b1a8811d601cb757f1e1ef6728d Mon Sep 17 00:00:00 2001 From: mdb-ad Date: Fri, 7 Feb 2025 13:11:23 -0800 Subject: [PATCH 03/33] Apply suggestions from code review Co-authored-by: Kevin Albertson --- .../tests/valid-pass/poc-queryable-encryption.yml | 8 ++++---- source/unified-test-format/unified-test-format.md | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/source/unified-test-format/tests/valid-pass/poc-queryable-encryption.yml b/source/unified-test-format/tests/valid-pass/poc-queryable-encryption.yml index 9a81a40ed5..19ad1f3e87 100644 --- a/source/unified-test-format/tests/valid-pass/poc-queryable-encryption.yml +++ b/source/unified-test-format/tests/valid-pass/poc-queryable-encryption.yml @@ -3,7 +3,8 @@ description: poc-queryable-encryption schemaVersion: "1.23" runOnRequirements: - - minServerVersion: "8.0" + - minServerVersion: "7.0" + csfle: true createEntities: - client: @@ -37,7 +38,7 @@ initialData: - databaseName: *database1Name collectionName: *collection1Name documents: - - _id: { $binary: { base64: EjRWeBI0mHYSNBI0VniQEg==, subType: "04" } } + - _id: &keyid { $binary: { base64: EjRWeBI0mHYSNBI0VniQEg==, subType: "04" } } keyMaterial: { $binary: { base64: sHe0kz57YW7v8g9VP9sf/+K1ex4JqKc5rf/URX3n3p8XdZ6+15uXPaSayC6adWbNxkFskuMCOifDoTT+rkqMtFkDclOy884RuGGtUysq3X7zkAWYTKi8QAfKkajvVbZl2y23UqgVasdQu3OVBQCrH/xY00nNAs/52e958nVjBuzQkSb1T8pKJAyjZsHJ60+FtnfafDZSTAIBJYn7UWBCwQ==, subType: "00" } } creationDate: { $date: { $numberLong: "1641024000000" } } updateDate: { $date: { $numberLong: "1641024000000" } } @@ -50,8 +51,7 @@ initialData: createOptions: encryptedFields: fields: - - keyId: - $binary: {'base64': 'EjRWeBI0mHYSNBI0VniQEg==', 'subType': '04'} + - keyId: *keyid path: 'encryptedInt' bsonType: 'int' queries: {'queryType': 'equality', 'contention': {'$numberLong': '0'}} diff --git a/source/unified-test-format/unified-test-format.md b/source/unified-test-format/unified-test-format.md index 193e6208b0..c5ba483136 100644 --- a/source/unified-test-format/unified-test-format.md +++ b/source/unified-test-format/unified-test-format.md @@ -538,7 +538,7 @@ The structure of this object is as follows: - `autoEncryptOpts`: Optional object with the following fields: - `kmsProviders`: The same as in [`clientEncryption`](#entity_clientEncryption). - - `keyVaultNamespace`: Optional, a namespace to the key vault collection. Defaults to "keyvault.datakeys". + - `keyVaultNamespace`: The same as in [`clientEncryption`](#entity_clientEncryption). - `bypassAutoEncryption`: Optional, a boolean to indicate whether or not auto encryption should be bypassed. Defaults to `false`. From 44d883cb2c1019b5689ad5e0f9561ef672f3e20a Mon Sep 17 00:00:00 2001 From: mdb-ad Date: Tue, 11 Feb 2025 17:45:09 -0800 Subject: [PATCH 04/33] poc test feedback --- .../valid-pass/poc-queryable-encryption.json | 27 +++++++------ .../valid-pass/poc-queryable-encryption.yml | 40 +++++++++---------- 2 files changed, 34 insertions(+), 33 deletions(-) diff --git a/source/unified-test-format/tests/valid-pass/poc-queryable-encryption.json b/source/unified-test-format/tests/valid-pass/poc-queryable-encryption.json index 4fa5a350e0..1b748f698e 100644 --- a/source/unified-test-format/tests/valid-pass/poc-queryable-encryption.json +++ b/source/unified-test-format/tests/valid-pass/poc-queryable-encryption.json @@ -3,7 +3,8 @@ "schemaVersion": "1.23", "runOnRequirements": [ { - "minServerVersion": "8.0" + "minServerVersion": "7.0", + "csfle": true } ], "createEntities": [ @@ -25,29 +26,29 @@ }, { "database": { - "id": "database0", + "id": "encryptedDB", "client": "client0", "databaseName": "poc-queryable-encryption" } }, { "collection": { - "id": "collection0", - "database": "database0", - "collectionName": "coll0" + "id": "encryptedColl", + "database": "encryptedDB", + "collectionName": "encrypted" } }, { "database": { - "id": "database1", + "id": "keyvaultDB", "client": "client0", "databaseName": "keyvault" } }, { "collection": { - "id": "collection1", - "database": "database0", + "id": "datakeysColl", + "database": "keyvaultDB", "collectionName": "datakeys" } } @@ -89,7 +90,7 @@ }, { "databaseName": "poc-queryable-encryption", - "collectionName": "coll0", + "collectionName": "encrypted", "documents": [], "createOptions": { "encryptedFields": { @@ -120,7 +121,7 @@ "description": "insert, replace, and find with queryable encryption", "operations": [ { - "object": "collection0", + "object": "encryptedColl", "name": "insertOne", "arguments": { "document": { @@ -130,7 +131,7 @@ } }, { - "object": "collection0", + "object": "encryptedColl", "name": "replaceOne", "arguments": { "filter": { @@ -142,7 +143,7 @@ } }, { - "object": "collection0", + "object": "encryptedColl", "name": "find", "arguments": { "filter": { @@ -159,7 +160,7 @@ ], "outcome": [ { - "collectionName": "coll0", + "collectionName": "encrypted", "databaseName": "poc-queryable-encryption", "documents": [ { diff --git a/source/unified-test-format/tests/valid-pass/poc-queryable-encryption.yml b/source/unified-test-format/tests/valid-pass/poc-queryable-encryption.yml index 19ad1f3e87..6e70956629 100644 --- a/source/unified-test-format/tests/valid-pass/poc-queryable-encryption.yml +++ b/source/unified-test-format/tests/valid-pass/poc-queryable-encryption.yml @@ -4,7 +4,7 @@ schemaVersion: "1.23" runOnRequirements: - minServerVersion: "7.0" - csfle: true + csfle: true createEntities: - client: @@ -17,26 +17,26 @@ createEntities: local: key: Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk - database: - id: &database0 database0 + id: &encryptedDB encryptedDB client: *client0 - databaseName: &database0Name poc-queryable-encryption + databaseName: &encryptedDBName poc-queryable-encryption - collection: - id: &collection0 collection0 - database: *database0 - collectionName: &collection0Name coll0 + id: &encryptedColl encryptedColl + database: *encryptedDB + collectionName: &encryptedCollName encrypted - database: - id: &database1 database1 + id: &keyvault keyvaultDB client: *client0 - databaseName: &database1Name keyvault + databaseName: &keyvaultName keyvault - collection: - id: &collection1 collection1 - database: *database0 - collectionName: &collection1Name datakeys + id: &datakeys datakeysColl + database: *keyvault + collectionName: &datakeysName datakeys initialData: - - databaseName: *database1Name - collectionName: *collection1Name + - databaseName: *keyvaultName + collectionName: *datakeysName documents: - _id: &keyid { $binary: { base64: EjRWeBI0mHYSNBI0VniQEg==, subType: "04" } } keyMaterial: { $binary: { base64: sHe0kz57YW7v8g9VP9sf/+K1ex4JqKc5rf/URX3n3p8XdZ6+15uXPaSayC6adWbNxkFskuMCOifDoTT+rkqMtFkDclOy884RuGGtUysq3X7zkAWYTKi8QAfKkajvVbZl2y23UqgVasdQu3OVBQCrH/xY00nNAs/52e958nVjBuzQkSb1T8pKJAyjZsHJ60+FtnfafDZSTAIBJYn7UWBCwQ==, subType: "00" } } @@ -45,8 +45,8 @@ initialData: status: 1 masterKey: provider: local - - databaseName: *database0Name - collectionName: *collection0Name + - databaseName: *encryptedDBName + collectionName: *encryptedCollName documents: [] createOptions: encryptedFields: @@ -59,18 +59,18 @@ initialData: tests: - description: insert, replace, and find with queryable encryption operations: - - object: *collection0 + - object: *encryptedColl name: insertOne arguments: document: _id: 1 encryptedInt: 11 - - object: *collection0 + - object: *encryptedColl name: replaceOne arguments: filter: { encryptedInt: 11 } replacement: { encryptedInt: 22 } - - object: *collection0 + - object: *encryptedColl name: find arguments: filter: { encryptedInt: 22 } @@ -78,7 +78,7 @@ tests: - _id: 1 encryptedInt: 22 outcome: - - collectionName: *collection0Name - databaseName: *database0Name + - collectionName: *encryptedCollName + databaseName: *encryptedDBName documents: - { _id: 1, encryptedInt: { $$type: binData }, __safeContent__: { $$type: array} } \ No newline at end of file From ad2fd55605c52e333c101c4637b151e1deef6f09 Mon Sep 17 00:00:00 2001 From: mdb-ad Date: Tue, 11 Feb 2025 17:50:43 -0800 Subject: [PATCH 05/33] make keyVaultNamespace required --- source/unified-test-format/schema-1.23.json | 1 + .../tests/valid-pass/poc-queryable-encryption.json | 1 + .../tests/valid-pass/poc-queryable-encryption.yml | 1 + 3 files changed, 3 insertions(+) diff --git a/source/unified-test-format/schema-1.23.json b/source/unified-test-format/schema-1.23.json index 6dd34a6cda..b0744eda8a 100644 --- a/source/unified-test-format/schema-1.23.json +++ b/source/unified-test-format/schema-1.23.json @@ -190,6 +190,7 @@ "autoEncryptOpts": { "type": "object", "required": [ + "keyVaultNamespace", "kmsProviders" ], "properties": { diff --git a/source/unified-test-format/tests/valid-pass/poc-queryable-encryption.json b/source/unified-test-format/tests/valid-pass/poc-queryable-encryption.json index 1b748f698e..2f055e30a2 100644 --- a/source/unified-test-format/tests/valid-pass/poc-queryable-encryption.json +++ b/source/unified-test-format/tests/valid-pass/poc-queryable-encryption.json @@ -16,6 +16,7 @@ "commandSucceededEvent" ], "autoEncryptOpts": { + "keyVaultNamespace": "keyvault.datakeys", "kmsProviders": { "local": { "key": "Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk" diff --git a/source/unified-test-format/tests/valid-pass/poc-queryable-encryption.yml b/source/unified-test-format/tests/valid-pass/poc-queryable-encryption.yml index 6e70956629..896bee2701 100644 --- a/source/unified-test-format/tests/valid-pass/poc-queryable-encryption.yml +++ b/source/unified-test-format/tests/valid-pass/poc-queryable-encryption.yml @@ -13,6 +13,7 @@ createEntities: - commandStartedEvent - commandSucceededEvent autoEncryptOpts: + keyVaultNamespace: keyvault.datakeys kmsProviders: local: key: Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk From 573aac1cc6131aa37d5acef0be00f67c37fcf233 Mon Sep 17 00:00:00 2001 From: mdb-ad Date: Tue, 11 Feb 2025 18:47:45 -0800 Subject: [PATCH 06/33] remaining auto encrypt options --- source/unified-test-format/schema-1.23.json | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/source/unified-test-format/schema-1.23.json b/source/unified-test-format/schema-1.23.json index b0744eda8a..fd092b0dc2 100644 --- a/source/unified-test-format/schema-1.23.json +++ b/source/unified-test-format/schema-1.23.json @@ -202,6 +202,21 @@ }, "kmsProviders": { "$ref": "#/definitions/kmsProviders" + }, + "schemaMap": { + "type": "object" + }, + "extraOptions": { + "type": "object" + }, + "encryptedFieldsMap": { + "type": "object" + }, + "bypassQueryAnalysis": { + "type": "boolean" + }, + "keyExpirationMS": { + "type": "integer" } } } From 112d5bd873343ff2c8570150cc875f6ae9062bcb Mon Sep 17 00:00:00 2001 From: mdb-ad Date: Wed, 12 Feb 2025 18:07:58 -0800 Subject: [PATCH 07/33] simple schema for schemaMap and encryptedFieldsMap --- source/unified-test-format/schema-1.23.json | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/source/unified-test-format/schema-1.23.json b/source/unified-test-format/schema-1.23.json index fd092b0dc2..d75bc40ba2 100644 --- a/source/unified-test-format/schema-1.23.json +++ b/source/unified-test-format/schema-1.23.json @@ -204,13 +204,19 @@ "$ref": "#/definitions/kmsProviders" }, "schemaMap": { - "type": "object" + "type": "object", + "additionalProperties": { + "type": "object" + } }, "extraOptions": { "type": "object" }, "encryptedFieldsMap": { - "type": "object" + "type": "object", + "additionalProperties": { + "type": "object" + } }, "bypassQueryAnalysis": { "type": "boolean" From f910f658835bf6f7481513c77e5aac7d2255957a Mon Sep 17 00:00:00 2001 From: mdb-ad Date: Fri, 14 Feb 2025 15:59:45 -0800 Subject: [PATCH 08/33] convert localSchema.yml --- .../tests/legacy/localSchema.yml | 65 ----------- .../tests/unified/localSchema.yml | 110 ++++++++++++++++++ .../valid-pass/poc-queryable-encryption.json | 4 - .../valid-pass/poc-queryable-encryption.yml | 3 - 4 files changed, 110 insertions(+), 72 deletions(-) delete mode 100644 source/client-side-encryption/tests/legacy/localSchema.yml create mode 100644 source/client-side-encryption/tests/unified/localSchema.yml diff --git a/source/client-side-encryption/tests/legacy/localSchema.yml b/source/client-side-encryption/tests/legacy/localSchema.yml deleted file mode 100644 index 89b4bd51d0..0000000000 --- a/source/client-side-encryption/tests/legacy/localSchema.yml +++ /dev/null @@ -1,65 +0,0 @@ -runOn: - - minServerVersion: "4.1.10" -database_name: &database_name "default" -collection_name: &collection_name "default" - -data: [] -# configure an empty schema -json_schema: {} -key_vault_data: [{'status': 1, '_id': {'$binary': {'base64': 'AAAAAAAAAAAAAAAAAAAAAA==', 'subType': '04'}}, 'masterKey': {'provider': 'aws', 'key': 'arn:aws:kms:us-east-1:579766882180:key/89fcc2c4-08b0-4bd9-9f25-e30687b580d0', 'region': 'us-east-1'}, 'updateDate': {'$date': {'$numberLong': '1552949630483'}}, 'keyMaterial': {'$binary': {'base64': 'AQICAHhQNmWG2CzOm1dq3kWLM+iDUZhEqnhJwH9wZVpuZ94A8gEqnsxXlR51T5EbEVezUqqKAAAAwjCBvwYJKoZIhvcNAQcGoIGxMIGuAgEAMIGoBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDHa4jo6yp0Z18KgbUgIBEIB74sKxWtV8/YHje5lv5THTl0HIbhSwM6EqRlmBiFFatmEWaeMk4tO4xBX65eq670I5TWPSLMzpp8ncGHMmvHqRajNBnmFtbYxN3E3/WjxmdbOOe+OXpnGJPcGsftc7cB2shRfA4lICPnE26+oVNXT6p0Lo20nY5XC7jyCO', 'subType': '00'}}, 'creationDate': {'$date': {'$numberLong': '1552949630483'}}, 'keyAltNames': ['altname', 'another_altname']}] - -tests: - - description: "A local schema should override" - clientOptions: - autoEncryptOpts: - schemaMap: - "default.default": {'properties': {'encrypted_w_altname': {'encrypt': {'keyId': '/altname', 'bsonType': 'string', 'algorithm': 'AEAD_AES_256_CBC_HMAC_SHA_512-Random'}}, 'encrypted_string': {'encrypt': {'keyId': [{'$binary': {'base64': 'AAAAAAAAAAAAAAAAAAAAAA==', 'subType': '04'}}], 'bsonType': 'string', 'algorithm': 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic'}}, 'random': {'encrypt': {'keyId': [{'$binary': {'base64': 'AAAAAAAAAAAAAAAAAAAAAA==', 'subType': '04'}}], 'bsonType': 'string', 'algorithm': 'AEAD_AES_256_CBC_HMAC_SHA_512-Random'}}, 'encrypted_string_equivalent': {'encrypt': {'keyId': [{'$binary': {'base64': 'AAAAAAAAAAAAAAAAAAAAAA==', 'subType': '04'}}], 'bsonType': 'string', 'algorithm': 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic'}}}, 'bsonType': 'object'} - kmsProviders: - aws: {} # Credentials filled in from environment. - operations: - - name: insertOne - arguments: - document: &doc0 { _id: 1, encrypted_string: "string0" } - - name: find - arguments: - filter: { _id: 1 } - result: [*doc0] - expectations: - # Then key is fetched from the key vault. - - command_started_event: - command: - find: datakeys - filter: {"$or": [{"_id": {"$in": [ {'$binary': {'base64': 'AAAAAAAAAAAAAAAAAAAAAA==', 'subType': '04'}} ] }}, {"keyAltNames": {"$in": []}}]} - $db: keyvault - readConcern: { level: "majority" } - command_name: find - - command_started_event: - command: - insert: *collection_name - documents: - - &doc0_encrypted { _id: 1, encrypted_string: {'$binary': {'base64': 'AQAAAAAAAAAAAAAAAAAAAAACwj+3zkv2VM+aTfk60RqhXq6a/77WlLwu/BxXFkL7EppGsju/m8f0x5kBDD3EZTtGALGXlym5jnpZAoSIkswHoA==', 'subType': '06'}} } - ordered: true - command_name: insert - - command_started_event: - command: - find: *collection_name - filter: { _id: 1 } - command_name: find - outcome: - collection: - # Outcome is checked using a separate MongoClient without auto encryption. - data: - - *doc0_encrypted - - description: "A local schema with no encryption is an error" - clientOptions: - autoEncryptOpts: - schemaMap: - "default.default": {'properties': {'test': {'bsonType': 'string'}}, 'bsonType': 'object', 'required': ['test']} - kmsProviders: - aws: {} # Credentials filled in from environment. - operations: - - name: insertOne - arguments: - document: { _id: 1, encrypted_string: "string0" } - result: - errorContains: "JSON schema keyword 'required' is only allowed with a remote schema" \ No newline at end of file diff --git a/source/client-side-encryption/tests/unified/localSchema.yml b/source/client-side-encryption/tests/unified/localSchema.yml new file mode 100644 index 0000000000..a8e9c23648 --- /dev/null +++ b/source/client-side-encryption/tests/unified/localSchema.yml @@ -0,0 +1,110 @@ +description: localSchema + +schemaVersion: "1.23" + +runOnRequirements: + - minServerVersion: "4.1.10" + - csfle: true + +createEntities: + - client: + id: &client0 client0 + autoEncryptOpts: + schemaMap: + "default.default": {'properties': {'encrypted_w_altname': {'encrypt': {'keyId': '/altname', 'bsonType': 'string', 'algorithm': 'AEAD_AES_256_CBC_HMAC_SHA_512-Random'}}, 'encrypted_string': {'encrypt': {'keyId': [{'$binary': {'base64': 'AAAAAAAAAAAAAAAAAAAAAA==', 'subType': '04'}}], 'bsonType': 'string', 'algorithm': 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic'}}, 'random': {'encrypt': {'keyId': [{'$binary': {'base64': 'AAAAAAAAAAAAAAAAAAAAAA==', 'subType': '04'}}], 'bsonType': 'string', 'algorithm': 'AEAD_AES_256_CBC_HMAC_SHA_512-Random'}}, 'encrypted_string_equivalent': {'encrypt': {'keyId': [{'$binary': {'base64': 'AAAAAAAAAAAAAAAAAAAAAA==', 'subType': '04'}}], 'bsonType': 'string', 'algorithm': 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic'}}}, 'bsonType': 'object'} + kmsProviders: + aws: {} # Credentials filled in from environment. + observeEvents: [ commandStartedEvent ] + - client: + id: &client1 client1 + autoEncryptOpts: + schemaMap: + "default.default": {'properties': {'test': {'bsonType': 'string'}}, 'bsonType': 'object', 'required': ['test']} + kmsProviders: + aws: {} # Credentials filled in from environment. + observeEvents: [ commandStartedEvent ] + - database: + id: &keyvault keyvaultDB + client: *client0 + databaseName: &keyvaultDBName keyvault + - collection: + id: &datakeys datakeysColl + database: *keyvault + collectionName: &datakeysCollName datakeys + - database: + id: &encryptedDB encryptedDB + client: *client0 + databaseName: &encryptedDBName default + - collection: + id: &encryptedColl encryptedColl + database: *encryptedDB + collectionName: &encryptedCollName default + # intentionally the same DB and collection name as encryptedDB/Coll + - database: + id: &encryptedDB2 encryptedDB2 + client: *client1 + databaseName: &encryptedDBName default + - collection: + id: &encryptedColl2 encryptedColl2 + database: *encryptedDB2 + collectionName: &encryptedCollName default + +initialData: + - databaseName: *keyvaultDBName + collectionName: *datakeysCollName + documents: + - {'status': 1, '_id': {'$binary': {'base64': 'AAAAAAAAAAAAAAAAAAAAAA==', 'subType': '04'}}, 'masterKey': {'provider': 'aws', 'key': 'arn:aws:kms:us-east-1:579766882180:key/89fcc2c4-08b0-4bd9-9f25-e30687b580d0', 'region': 'us-east-1'}, 'updateDate': {'$date': {'$numberLong': '1552949630483'}}, 'keyMaterial': {'$binary': {'base64': 'AQICAHhQNmWG2CzOm1dq3kWLM+iDUZhEqnhJwH9wZVpuZ94A8gEqnsxXlR51T5EbEVezUqqKAAAAwjCBvwYJKoZIhvcNAQcGoIGxMIGuAgEAMIGoBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDHa4jo6yp0Z18KgbUgIBEIB74sKxWtV8/YHje5lv5THTl0HIbhSwM6EqRlmBiFFatmEWaeMk4tO4xBX65eq670I5TWPSLMzpp8ncGHMmvHqRajNBnmFtbYxN3E3/WjxmdbOOe+OXpnGJPcGsftc7cB2shRfA4lICPnE26+oVNXT6p0Lo20nY5XC7jyCO', 'subType': '00'}}, 'creationDate': {'$date': {'$numberLong': '1552949630483'}}, 'keyAltNames': ['altname', 'another_altname']} + - databaseName: *encryptedDBName + collectionName: *encryptedCollName + documents: [] + createOptions: + validator: {} + +tests: + - description: "A local schema should override" + operations: + - object: *encryptedColl + name: insertOne + arguments: + document: &doc0 { _id: 1, encrypted_string: "string0" } + - object: *encryptedColl + name: find + arguments: + filter: { _id: 1 } + expectResult: [*doc0] + expectEvents: + # Then key is fetched from the key vault. + - client: *client0 + events: + - commandStartedEvent: + databaseName: *keyvaultDBName + commandName: find + command: + find: *datakeysCollName + filter: {"$or": [{"_id": {"$in": [ {'$binary': {'base64': 'AAAAAAAAAAAAAAAAAAAAAA==', 'subType': '04'}} ] }}, {"keyAltNames": {"$in": []}}]} + readConcern: { level: "majority" } + - commandStartedEvent: + commandName: insert + command: + insert: *encryptedCollName + documents: + - &doc0_encrypted { _id: 1, encrypted_string: {'$binary': {'base64': 'AQAAAAAAAAAAAAAAAAAAAAACwj+3zkv2VM+aTfk60RqhXq6a/77WlLwu/BxXFkL7EppGsju/m8f0x5kBDD3EZTtGALGXlym5jnpZAoSIkswHoA==', 'subType': '06'}} } + ordered: true + - commandStartedEvent: + commandName: find + command: + find: *encryptedCollName + filter: { _id: 1 } + outcome: + - collectionName: *encryptedCollName + databaseName: *encryptedDBName + documents: + - *doc0_encrypted + - description: "A local schema with no encryption is an error" + operations: + - object: *encryptedColl2 + name: insertOne + arguments: + document: &doc0 { _id: 1, encrypted_string: "string0" } + expectError: + isClientError: true \ No newline at end of file diff --git a/source/unified-test-format/tests/valid-pass/poc-queryable-encryption.json b/source/unified-test-format/tests/valid-pass/poc-queryable-encryption.json index 2f055e30a2..2d660c1729 100644 --- a/source/unified-test-format/tests/valid-pass/poc-queryable-encryption.json +++ b/source/unified-test-format/tests/valid-pass/poc-queryable-encryption.json @@ -11,10 +11,6 @@ { "client": { "id": "client0", - "observeEvents": [ - "commandStartedEvent", - "commandSucceededEvent" - ], "autoEncryptOpts": { "keyVaultNamespace": "keyvault.datakeys", "kmsProviders": { diff --git a/source/unified-test-format/tests/valid-pass/poc-queryable-encryption.yml b/source/unified-test-format/tests/valid-pass/poc-queryable-encryption.yml index 896bee2701..1ec34beea7 100644 --- a/source/unified-test-format/tests/valid-pass/poc-queryable-encryption.yml +++ b/source/unified-test-format/tests/valid-pass/poc-queryable-encryption.yml @@ -9,9 +9,6 @@ runOnRequirements: createEntities: - client: id: &client0 client0 - observeEvents: - - commandStartedEvent - - commandSucceededEvent autoEncryptOpts: keyVaultNamespace: keyvault.datakeys kmsProviders: From 364eb4c8a69f243c7b622fdb2512024c49a54cfb Mon Sep 17 00:00:00 2001 From: mdb-ad Date: Fri, 14 Feb 2025 16:23:42 -0800 Subject: [PATCH 09/33] convert fle2v2-EncryptedFields-vs-EncryptedFieldsMap --- ...-EncryptedFields-vs-EncryptedFieldsMap.yml | 79 ----- ...EncryptedFields-vs-EncryptedFieldsMap.json | 272 ++++++++++++++++++ ...-EncryptedFields-vs-EncryptedFieldsMap.yml | 119 ++++++++ 3 files changed, 391 insertions(+), 79 deletions(-) delete mode 100644 source/client-side-encryption/tests/legacy/fle2v2-EncryptedFields-vs-EncryptedFieldsMap.yml create mode 100644 source/client-side-encryption/tests/unified/fle2v2-EncryptedFields-vs-EncryptedFieldsMap.json create mode 100644 source/client-side-encryption/tests/unified/fle2v2-EncryptedFields-vs-EncryptedFieldsMap.yml diff --git a/source/client-side-encryption/tests/legacy/fle2v2-EncryptedFields-vs-EncryptedFieldsMap.yml b/source/client-side-encryption/tests/legacy/fle2v2-EncryptedFields-vs-EncryptedFieldsMap.yml deleted file mode 100644 index 8767132e62..0000000000 --- a/source/client-side-encryption/tests/legacy/fle2v2-EncryptedFields-vs-EncryptedFieldsMap.yml +++ /dev/null @@ -1,79 +0,0 @@ -# Requires libmongocrypt 1.8.0. -runOn: - - minServerVersion: "7.0.0" - # Skip QEv2 (also referred to as FLE2v2) tests on Serverless. Unskip once Serverless enables the QEv2 protocol. - # FLE 2 Encrypted collections are not supported on standalone. - topology: [ "replicaset", "sharded", "load-balanced" ] -database_name: &database_name "default" -collection_name: &collection_name "default" -data: [] -encrypted_fields: {'fields': [{'keyId': {'$binary': {'base64': 'EjRWeBI0mHYSNBI0VniQEg==', 'subType': '04'}}, 'path': 'encryptedIndexed', 'bsonType': 'string', 'queries': {'queryType': 'equality', 'contention': {'$numberLong': '0'}}}, {'keyId': {'$binary': {'base64': 'q83vqxI0mHYSNBI0VniQEg==', 'subType': '04'}}, 'path': 'encryptedUnindexed', 'bsonType': 'string'}]} -key_vault_data: [ {'_id': {'$binary': {'base64': 'q83vqxI0mHYSNBI0VniQEg==', 'subType': '04'}}, 'keyMaterial': {'$binary': {'base64': 'HBk9BWihXExNDvTp1lUxOuxuZK2Pe2ZdVdlsxPEBkiO1bS4mG5NNDsQ7zVxJAH8BtdOYp72Ku4Y3nwc0BUpIKsvAKX4eYXtlhv5zUQxWdeNFhg9qK7qb8nqhnnLeT0f25jFSqzWJoT379hfwDeu0bebJHr35QrJ8myZdPMTEDYF08QYQ48ShRBli0S+QzBHHAQiM2iJNr4svg2WR8JSeWQ==', 'subType': '00'}}, 'creationDate': {'$date': {'$numberLong': '1648914851981'}}, 'updateDate': {'$date': {'$numberLong': '1648914851981'}}, 'status': {'$numberInt': '0'}, 'masterKey': {'provider': 'local'}}] -tests: - - description: "encryptedFieldsMap is preferred over remote encryptedFields" - clientOptions: - autoEncryptOpts: - kmsProviders: - local: {'key': {'$binary': {'base64': 'Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk', 'subType': '00'}}} - encryptedFieldsMap: { - "default.default": { - "fields": [] - } - } - operations: - # EncryptedFieldsMap overrides remote encryptedFields. - # Automatic encryption does not occur on encryptedUnindexed. The value is validated on the server. - - name: insertOne - arguments: - document: &doc0 { - _id: 1, - encryptedUnindexed: { - "$binary": { - "base64": "BqvN76sSNJh2EjQSNFZ4kBICTQaVZPWgXp41I7mPV1rLFTtw1tXzjcdSEyxpKKqujlko5TeizkB9hHQ009dVY1+fgIiDcefh+eQrm3CkhQ==", - "subType": "06" - } - } - } - - name: find - arguments: - filter: { "_id": 1 } - result: [{"_id": 1, "encryptedUnindexed": "value123" }] - expectations: - - command_started_event: - command: - insert: *collection_name - documents: - - *doc0 - ordered: true - command_name: insert - - command_started_event: - command: - find: *collection_name - filter: { "_id": 1} - command_name: find - - command_started_event: - command: - find: datakeys - filter: { - "$or": [ - { - "_id": { - "$in": [ - {'$binary': {'base64': 'q83vqxI0mHYSNBI0VniQEg==', 'subType': '04'}} - ] - } - }, - { - "keyAltNames": { - "$in": [] - } - } - ] - } - $db: keyvault - readConcern: { level: "majority" } - command_name: find - outcome: - collection: - data: - - *doc0 \ No newline at end of file diff --git a/source/client-side-encryption/tests/unified/fle2v2-EncryptedFields-vs-EncryptedFieldsMap.json b/source/client-side-encryption/tests/unified/fle2v2-EncryptedFields-vs-EncryptedFieldsMap.json new file mode 100644 index 0000000000..6c26847526 --- /dev/null +++ b/source/client-side-encryption/tests/unified/fle2v2-EncryptedFields-vs-EncryptedFieldsMap.json @@ -0,0 +1,272 @@ +{ + "description": "fle2v2-EncryptedFields-vs-EncryptedFieldsMap", + "schemaVersion": "1.23", + "runOnRequirements": [ + { + "minServerVersion": "7.0.0", + "topologies": [ + "replicaset", + "sharded", + "load-balanced" + ] + } + ], + "createEntities": [ + { + "client": { + "id": "client0", + "autoEncryptOpts": { + "kmsProviders": { + "local": { + "key": { + "$binary": { + "base64": "Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk", + "subType": "00" + } + } + } + }, + "encryptedFieldsMap": { + "default.default": { + "fields": [] + } + } + }, + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "database": { + "id": "keyvaultDB", + "client": "client0", + "databaseName": "keyvault" + } + }, + { + "collection": { + "id": "datakeysColl", + "database": "keyvaultDB", + "collectionName": "datakeys" + } + }, + { + "database": { + "id": "encryptedDB", + "client": "client0", + "databaseName": "default" + } + }, + { + "collection": { + "id": "encryptedColl", + "database": "encryptedDB", + "collectionName": "default" + } + } + ], + "initialData": [ + { + "databaseName": "keyvault", + "collectionName": "datakeys", + "documents": [ + { + "_id": { + "$binary": { + "base64": "q83vqxI0mHYSNBI0VniQEg==", + "subType": "04" + } + }, + "keyMaterial": { + "$binary": { + "base64": "HBk9BWihXExNDvTp1lUxOuxuZK2Pe2ZdVdlsxPEBkiO1bS4mG5NNDsQ7zVxJAH8BtdOYp72Ku4Y3nwc0BUpIKsvAKX4eYXtlhv5zUQxWdeNFhg9qK7qb8nqhnnLeT0f25jFSqzWJoT379hfwDeu0bebJHr35QrJ8myZdPMTEDYF08QYQ48ShRBli0S+QzBHHAQiM2iJNr4svg2WR8JSeWQ==", + "subType": "00" + } + }, + "creationDate": { + "$date": { + "$numberLong": "1648914851981" + } + }, + "updateDate": { + "$date": { + "$numberLong": "1648914851981" + } + }, + "status": { + "$numberInt": "0" + }, + "masterKey": { + "provider": "local" + } + } + ] + }, + { + "databaseName": "default", + "collectionName": "default", + "documents": [], + "createOptions": { + "encryptedFields": { + "fields": [ + { + "keyId": { + "$binary": { + "base64": "EjRWeBI0mHYSNBI0VniQEg==", + "subType": "04" + } + }, + "path": "encryptedIndexed", + "bsonType": "string", + "queries": { + "queryType": "equality", + "contention": { + "$numberLong": "0" + } + } + }, + { + "keyId": { + "$binary": { + "base64": "q83vqxI0mHYSNBI0VniQEg==", + "subType": "04" + } + }, + "path": "encryptedUnindexed", + "bsonType": "string" + } + ] + } + } + } + ], + "tests": [ + { + "description": "encryptedFieldsMap is preferred over remote encryptedFields", + "operations": [ + { + "object": "encryptedColl", + "name": "insertOne", + "arguments": { + "document": { + "_id": 1, + "encryptedUnindexed": { + "$binary": { + "base64": "BqvN76sSNJh2EjQSNFZ4kBICTQaVZPWgXp41I7mPV1rLFTtw1tXzjcdSEyxpKKqujlko5TeizkB9hHQ009dVY1+fgIiDcefh+eQrm3CkhQ==", + "subType": "06" + } + } + } + } + }, + { + "object": "encryptedColl", + "name": "find", + "arguments": { + "filter": { + "_id": 1 + } + }, + "expectResult": [ + { + "_id": 1, + "encryptedUnindexed": "value123" + } + ] + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "databaseName": "default", + "commandName": "insert", + "command": { + "insert": "default", + "documents": [ + { + "_id": 1, + "encryptedUnindexed": { + "$binary": { + "base64": "BqvN76sSNJh2EjQSNFZ4kBICTQaVZPWgXp41I7mPV1rLFTtw1tXzjcdSEyxpKKqujlko5TeizkB9hHQ009dVY1+fgIiDcefh+eQrm3CkhQ==", + "subType": "06" + } + } + } + ], + "ordered": true + } + } + }, + { + "commandStartedEvent": { + "databaseName": "default", + "commandName": "find", + "command": { + "find": "default", + "filter": { + "_id": 1 + } + } + } + }, + { + "commandStartedEvent": { + "databaseName": "keyvault", + "commandName": "find", + "command": { + "find": "datakeys", + "filter": { + "$or": [ + { + "_id": { + "$in": [ + { + "$binary": { + "base64": "q83vqxI0mHYSNBI0VniQEg==", + "subType": "04" + } + } + ] + } + }, + { + "keyAltNames": { + "$in": [] + } + } + ] + }, + "$db": "keyvault", + "readConcern": { + "level": "majority" + } + } + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "default", + "databaseName": "default", + "documents": [ + { + "_id": 1, + "encryptedUnindexed": { + "$binary": { + "base64": "BqvN76sSNJh2EjQSNFZ4kBICTQaVZPWgXp41I7mPV1rLFTtw1tXzjcdSEyxpKKqujlko5TeizkB9hHQ009dVY1+fgIiDcefh+eQrm3CkhQ==", + "subType": "06" + } + } + } + ] + } + ] + } + ] +} diff --git a/source/client-side-encryption/tests/unified/fle2v2-EncryptedFields-vs-EncryptedFieldsMap.yml b/source/client-side-encryption/tests/unified/fle2v2-EncryptedFields-vs-EncryptedFieldsMap.yml new file mode 100644 index 0000000000..8655afe5a9 --- /dev/null +++ b/source/client-side-encryption/tests/unified/fle2v2-EncryptedFields-vs-EncryptedFieldsMap.yml @@ -0,0 +1,119 @@ +description: fle2v2-EncryptedFields-vs-EncryptedFieldsMap + +schemaVersion: "1.23" + +runOnRequirements: + - minServerVersion: "7.0.0" + # Skip QEv2 (also referred to as FLE2v2) tests on Serverless. Unskip once Serverless enables the QEv2 protocol. + # FLE 2 Encrypted collections are not supported on standalone. + topologies: [ "replicaset", "sharded", "load-balanced" ] + +createEntities: + - client: + id: &client0 client0 + autoEncryptOpts: + kmsProviders: + local: {'key': {'$binary': {'base64': 'Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk', 'subType': '00'}}} + encryptedFieldsMap: { + "default.default": { + "fields": [] + } + } + observeEvents: [ commandStartedEvent ] + - database: + id: &keyvault keyvaultDB + client: *client0 + databaseName: &keyvaultDBName keyvault + - collection: + id: &datakeys datakeysColl + database: *keyvault + collectionName: &datakeysCollName datakeys + - database: + id: &encryptedDB encryptedDB + client: *client0 + databaseName: &encryptedDBName default + - collection: + id: &encryptedColl encryptedColl + database: *encryptedDB + collectionName: &encryptedCollName default + # intentionally the same DB and collection name as encryptedDB/Coll + +initialData: + - databaseName: *keyvaultDBName + collectionName: *datakeysCollName + documents: + - {'_id': {'$binary': {'base64': 'q83vqxI0mHYSNBI0VniQEg==', 'subType': '04'}}, 'keyMaterial': {'$binary': {'base64': 'HBk9BWihXExNDvTp1lUxOuxuZK2Pe2ZdVdlsxPEBkiO1bS4mG5NNDsQ7zVxJAH8BtdOYp72Ku4Y3nwc0BUpIKsvAKX4eYXtlhv5zUQxWdeNFhg9qK7qb8nqhnnLeT0f25jFSqzWJoT379hfwDeu0bebJHr35QrJ8myZdPMTEDYF08QYQ48ShRBli0S+QzBHHAQiM2iJNr4svg2WR8JSeWQ==', 'subType': '00'}}, 'creationDate': {'$date': {'$numberLong': '1648914851981'}}, 'updateDate': {'$date': {'$numberLong': '1648914851981'}}, 'status': {'$numberInt': '0'}, 'masterKey': {'provider': 'local'}} + - databaseName: *encryptedDBName + collectionName: *encryptedCollName + documents: [] + createOptions: + encryptedFields: {'fields': [{'keyId': {'$binary': {'base64': 'EjRWeBI0mHYSNBI0VniQEg==', 'subType': '04'}}, 'path': 'encryptedIndexed', 'bsonType': 'string', 'queries': {'queryType': 'equality', 'contention': {'$numberLong': '0'}}}, {'keyId': {'$binary': {'base64': 'q83vqxI0mHYSNBI0VniQEg==', 'subType': '04'}}, 'path': 'encryptedUnindexed', 'bsonType': 'string'}]} + +tests: + - description: "encryptedFieldsMap is preferred over remote encryptedFields" + operations: + # EncryptedFieldsMap overrides remote encryptedFields. + # Automatic encryption does not occur on encryptedUnindexed. The value is validated on the server. + - object: *encryptedColl + name: insertOne + arguments: + document: &doc0 { + _id: 1, + encryptedUnindexed: { + "$binary": { + "base64": "BqvN76sSNJh2EjQSNFZ4kBICTQaVZPWgXp41I7mPV1rLFTtw1tXzjcdSEyxpKKqujlko5TeizkB9hHQ009dVY1+fgIiDcefh+eQrm3CkhQ==", + "subType": "06" + } + } + } + - object: *encryptedColl + name: find + arguments: + filter: { "_id": 1 } + expectResult: + - {"_id": 1, "encryptedUnindexed": "value123" } + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + databaseName: *encryptedDBName + commandName: insert + command: + insert: *encryptedCollName + documents: + - *doc0 + ordered: true + - commandStartedEvent: + databaseName: *encryptedDBName + commandName: find + command: + find: *encryptedCollName + filter: { "_id": 1} + - commandStartedEvent: + databaseName: *keyvaultDBName + commandName: find + command: + find: *datakeysCollName + filter: { + "$or": [ + { + "_id": { + "$in": [ + {'$binary': {'base64': 'q83vqxI0mHYSNBI0VniQEg==', 'subType': '04'}} + ] + } + }, + { + "keyAltNames": { + "$in": [] + } + } + ] + } + $db: *keyvaultDBName + readConcern: { level: "majority" } + outcome: + - collectionName: *encryptedCollName + databaseName: *encryptedDBName + documents: + - *doc0 \ No newline at end of file From 01505fc1e6a64d4bd2e27487d2f54910af1a1e4e Mon Sep 17 00:00:00 2001 From: mdb-ad Date: Fri, 14 Feb 2025 17:04:37 -0800 Subject: [PATCH 10/33] document auto encrypt fields --- source/unified-test-format/unified-test-format.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/source/unified-test-format/unified-test-format.md b/source/unified-test-format/unified-test-format.md index c5ba483136..adc3612f5d 100644 --- a/source/unified-test-format/unified-test-format.md +++ b/source/unified-test-format/unified-test-format.md @@ -541,6 +541,11 @@ The structure of this object is as follows: - `keyVaultNamespace`: The same as in [`clientEncryption`](#entity_clientEncryption). - `bypassAutoEncryption`: Optional, a boolean to indicate whether or not auto encryption should be bypassed. Defaults to `false`. + - `schemaMap`: Optional object. Maps namespaces to encrypted fields. + - `encryptedFieldsMap`: Optional object. Specifies which fields to automatically encrypt and the types of queries allowed on those fields. + - `extraOptions`: Optional object. Configuration options for the encryption library. + - `bypassQueryAnalysis`: Optional. Disables analysis of outgoing commands. Defaults to `false`. + - `keyExpirationMS`: Optional integer. Sets how long data encryption keys are cached. Defaults to 60,000. From 87d3c41aabb1c65fb390d0489b1f1e81c0d58ef7 Mon Sep 17 00:00:00 2001 From: mdb-ad Date: Mon, 24 Feb 2025 00:16:28 -0800 Subject: [PATCH 11/33] client bulkWrite test --- .../tests/unified/client-bulkWrite-qe.json | 272 ++++++++++++++++++ .../tests/unified/client-bulkWrite-qe.yml | 112 ++++++++ 2 files changed, 384 insertions(+) create mode 100644 source/crud/tests/unified/client-bulkWrite-qe.json create mode 100644 source/crud/tests/unified/client-bulkWrite-qe.yml diff --git a/source/crud/tests/unified/client-bulkWrite-qe.json b/source/crud/tests/unified/client-bulkWrite-qe.json new file mode 100644 index 0000000000..4d73d01351 --- /dev/null +++ b/source/crud/tests/unified/client-bulkWrite-qe.json @@ -0,0 +1,272 @@ +{ + "description": "client bulkWrite with queryable encryption", + "schemaVersion": "1.23", + "runOnRequirements": [ + { + "minServerVersion": "8.0", + "serverless": "forbid" + } + ], + "createEntities": [ + { + "client": { + "id": "client0", + "observeEvents": [ + "commandStartedEvent", + "commandSucceededEvent" + ], + "autoEncryptOpts": { + "kmsProviders": { + "local": { + "key": "Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk" + } + } + } + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "crud-tests" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "coll0" + } + }, + { + "database": { + "id": "database1", + "client": "client0", + "databaseName": "keyvault" + } + }, + { + "collection": { + "id": "collection1", + "database": "database0", + "collectionName": "datakeys" + } + } + ], + "initialData": [ + { + "databaseName": "keyvault", + "collectionName": "datakeys", + "documents": [ + { + "_id": { + "$binary": { + "base64": "EjRWeBI0mHYSNBI0VniQEg==", + "subType": "04" + } + }, + "keyAltNames": [ + "local_key" + ], + "keyMaterial": { + "$binary": { + "base64": "sHe0kz57YW7v8g9VP9sf/+K1ex4JqKc5rf/URX3n3p8XdZ6+15uXPaSayC6adWbNxkFskuMCOifDoTT+rkqMtFkDclOy884RuGGtUysq3X7zkAWYTKi8QAfKkajvVbZl2y23UqgVasdQu3OVBQCrH/xY00nNAs/52e958nVjBuzQkSb1T8pKJAyjZsHJ60+FtnfafDZSTAIBJYn7UWBCwQ==", + "subType": "00" + } + }, + "creationDate": { + "$date": { + "$numberLong": "1641024000000" + } + }, + "updateDate": { + "$date": { + "$numberLong": "1641024000000" + } + }, + "status": 1, + "masterKey": { + "provider": "local" + } + } + ] + }, + { + "databaseName": "crud-tests", + "collectionName": "coll0", + "documents": [], + "createOptions": { + "encryptedFields": { + "fields": [ + { + "keyId": { + "$binary": { + "base64": "EjRWeBI0mHYSNBI0VniQEg==", + "subType": "04" + } + }, + "path": "encryptedInt", + "bsonType": "int", + "queries": { + "queryType": "equality", + "contention": { + "$numberLong": "0" + } + } + } + ] + } + } + } + ], + "_yamlAnchors": { + "namespace": "crud-tests.coll0" + }, + "tests": [ + { + "description": "client bulkWrite QE replaceOne", + "operations": [ + { + "object": "collection0", + "name": "insertMany", + "arguments": { + "documents": [ + { + "_id": 1, + "encryptedInt": 11 + }, + { + "_id": 2, + "encryptedInt": 22 + }, + { + "_id": 3, + "encryptedInt": 33 + } + ] + } + }, + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "models": [ + { + "replaceOne": { + "namespace": "crud-tests.coll0", + "filter": { + "encryptedInt": { + "$eq": 11 + } + }, + "replacement": { + "encryptedInt": 44 + } + } + } + ] + }, + "expectResult": { + "insertedCount": 0, + "upsertedCount": 0, + "matchedCount": 1, + "modifiedCount": 1, + "deletedCount": 0 + } + }, + { + "object": "collection0", + "name": "find", + "arguments": { + "filter": { + "encryptedInt": 44 + } + }, + "expectResult": [ + { + "_id": 1, + "encryptedInt": 44 + } + ] + } + ], + "outcome": [ + { + "collectionName": "coll0", + "databaseName": "crud-tests", + "documents": [ + { + "_id": 1, + "encryptedInt": { + "$$type": "binData" + }, + "__safeContent__": { + "$$type": "array" + } + }, + { + "_id": 2, + "encryptedInt": { + "$$type": "binData" + }, + "__safeContent__": { + "$$type": "array" + } + }, + { + "_id": 3, + "encryptedInt": { + "$$type": "binData" + }, + "__safeContent__": { + "$$type": "array" + } + } + ] + } + ] + }, + { + "description": "client bulkWrite QE with multiple replace fails", + "operations": [ + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "models": [ + { + "replaceOne": { + "namespace": "crud-tests.coll0", + "filter": { + "encryptedInt": { + "$eq": 11 + } + }, + "replacement": { + "encryptedInt": 44 + } + } + }, + { + "replaceOne": { + "namespace": "crud-tests.coll0", + "filter": { + "encryptedInt": { + "$eq": 22 + } + }, + "replacement": { + "encryptedInt": 44 + } + } + } + ] + }, + "expectError": { + "isError": true + } + } + ] + } + ] +} diff --git a/source/crud/tests/unified/client-bulkWrite-qe.yml b/source/crud/tests/unified/client-bulkWrite-qe.yml new file mode 100644 index 0000000000..fce623a586 --- /dev/null +++ b/source/crud/tests/unified/client-bulkWrite-qe.yml @@ -0,0 +1,112 @@ +description: client bulkWrite with queryable encryption + +schemaVersion: "1.23" + +runOnRequirements: + - minServerVersion: "8.0" + serverless: forbid # Serverless does not support bulkWrite: CLOUDP-256344. + +createEntities: + - client: + id: &client0 client0 + observeEvents: + - commandStartedEvent + - commandSucceededEvent + autoEncryptOpts: + kmsProviders: + local: + key: Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk + - database: + id: &database0 database0 + client: *client0 + databaseName: &database0Name crud-tests + - collection: + id: &collection0 collection0 + database: *database0 + collectionName: &collection0Name coll0 + - database: + id: &database1 database1 + client: *client0 + databaseName: &database1Name keyvault + - collection: + id: &collection1 collection1 + database: *database0 + collectionName: &collection1Name datakeys + + +initialData: + - databaseName: *database1Name + collectionName: *collection1Name + documents: + - _id: &local_key_id { $binary: { base64: EjRWeBI0mHYSNBI0VniQEg==, subType: "04" } } + keyAltNames: ["local_key"] + keyMaterial: { $binary: { base64: sHe0kz57YW7v8g9VP9sf/+K1ex4JqKc5rf/URX3n3p8XdZ6+15uXPaSayC6adWbNxkFskuMCOifDoTT+rkqMtFkDclOy884RuGGtUysq3X7zkAWYTKi8QAfKkajvVbZl2y23UqgVasdQu3OVBQCrH/xY00nNAs/52e958nVjBuzQkSb1T8pKJAyjZsHJ60+FtnfafDZSTAIBJYn7UWBCwQ==, subType: "00" } } + creationDate: { $date: { $numberLong: "1641024000000" } } + updateDate: { $date: { $numberLong: "1641024000000" } } + status: 1 + masterKey: &local_masterkey + provider: local + - databaseName: *database0Name + collectionName: *collection0Name + documents: [] + createOptions: + encryptedFields: &encrypted_fields {'fields': [{'keyId': {'$binary': {'base64': 'EjRWeBI0mHYSNBI0VniQEg==', 'subType': '04'}}, 'path': 'encryptedInt', 'bsonType': 'int', 'queries': {'queryType': 'equality', 'contention': {'$numberLong': '0'}}}]} + +_yamlAnchors: + namespace: &namespace "crud-tests.coll0" + +tests: + - description: client bulkWrite QE replaceOne + operations: + - object: *collection0 + name: insertMany + arguments: + documents: + - { _id: 1, encryptedInt: 11 } + - { _id: 2, encryptedInt: 22 } + - { _id: 3, encryptedInt: 33 } + - object: *client0 + name: clientBulkWrite + arguments: + models: + - replaceOne: + namespace: *namespace + filter: { encryptedInt: { $eq: 11 } } + replacement: { encryptedInt: 44 } + expectResult: + insertedCount: 0 + upsertedCount: 0 + matchedCount: 1 + modifiedCount: 1 + deletedCount: 0 + - object: *collection0 + name: find + arguments: + filter: { encryptedInt: 44 } + expectResult: + - _id: 1 + encryptedInt: 44 + outcome: + - collectionName: *collection0Name + databaseName: *database0Name + documents: + - { _id: 1, encryptedInt: { $$type: binData }, __safeContent__: { $$type: array} } + - { _id: 2, encryptedInt: { $$type: binData }, __safeContent__: { $$type: array} } + - { _id: 3, encryptedInt: { $$type: binData }, __safeContent__: { $$type: array} } + - description: client bulkWrite QE with multiple replace fails + operations: + - object: *client0 + name: clientBulkWrite + arguments: + models: + - replaceOne: + namespace: *namespace + filter: { encryptedInt: { $eq: 11 } } + replacement: { encryptedInt: 44 } + - replaceOne: + namespace: *namespace + filter: { encryptedInt: { $eq: 22 } } + replacement: { encryptedInt: 44 } + expectError: + # Expect error from mongocryptd or crypt_shared + isError: true From bd7a0fc92b236fb38db6a9e636d3b1c5cc321ff7 Mon Sep 17 00:00:00 2001 From: mdb-ad <198671546+mdb-ad@users.noreply.github.com> Date: Fri, 18 Apr 2025 11:00:18 -0700 Subject: [PATCH 12/33] keyvault namespace --- source/crud/tests/unified/client-bulkWrite-qe.json | 1 + source/crud/tests/unified/client-bulkWrite-qe.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/source/crud/tests/unified/client-bulkWrite-qe.json b/source/crud/tests/unified/client-bulkWrite-qe.json index 4d73d01351..bede25da0d 100644 --- a/source/crud/tests/unified/client-bulkWrite-qe.json +++ b/source/crud/tests/unified/client-bulkWrite-qe.json @@ -16,6 +16,7 @@ "commandSucceededEvent" ], "autoEncryptOpts": { + "keyVaultNamespace": "keyvault.datakeys", "kmsProviders": { "local": { "key": "Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk" diff --git a/source/crud/tests/unified/client-bulkWrite-qe.yml b/source/crud/tests/unified/client-bulkWrite-qe.yml index fce623a586..d8ebbc3f4b 100644 --- a/source/crud/tests/unified/client-bulkWrite-qe.yml +++ b/source/crud/tests/unified/client-bulkWrite-qe.yml @@ -13,6 +13,7 @@ createEntities: - commandStartedEvent - commandSucceededEvent autoEncryptOpts: + keyVaultNamespace: keyvault.datakeys kmsProviders: local: key: Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk From b2893e776dfadab1b28cd6c7c059b06b9de4c2ee Mon Sep 17 00:00:00 2001 From: mdb-ad <198671546+mdb-ad@users.noreply.github.com> Date: Wed, 23 Apr 2025 20:07:37 -0700 Subject: [PATCH 13/33] use unencrypted client to check collection contents --- .../tests/unified/client-bulkWrite-qe.json | 35 +++++++++++++++---- .../tests/unified/client-bulkWrite-qe.yml | 25 +++++++++---- 2 files changed, 48 insertions(+), 12 deletions(-) diff --git a/source/crud/tests/unified/client-bulkWrite-qe.json b/source/crud/tests/unified/client-bulkWrite-qe.json index bede25da0d..dad3f3950a 100644 --- a/source/crud/tests/unified/client-bulkWrite-qe.json +++ b/source/crud/tests/unified/client-bulkWrite-qe.json @@ -39,6 +39,14 @@ "collectionName": "coll0" } }, + { + "client": { + "id": "client1", + "observeEvents": [ + "commandStartedEvent" + ] + } + }, { "database": { "id": "database1", @@ -52,6 +60,20 @@ "database": "database0", "collectionName": "datakeys" } + }, + { + "database": { + "id": "database2", + "client": "client1", + "databaseName": "crud-tests" + } + }, + { + "collection": { + "id": "collection2", + "database": "database2", + "collectionName": "coll0" + } } ], "initialData": [ @@ -189,13 +211,14 @@ "encryptedInt": 44 } ] - } - ], - "outcome": [ + }, { - "collectionName": "coll0", - "databaseName": "crud-tests", - "documents": [ + "object": "collection2", + "name": "find", + "arguments": { + "filter": {} + }, + "expectResult": [ { "_id": 1, "encryptedInt": { diff --git a/source/crud/tests/unified/client-bulkWrite-qe.yml b/source/crud/tests/unified/client-bulkWrite-qe.yml index d8ebbc3f4b..3c29884932 100644 --- a/source/crud/tests/unified/client-bulkWrite-qe.yml +++ b/source/crud/tests/unified/client-bulkWrite-qe.yml @@ -25,6 +25,10 @@ createEntities: id: &collection0 collection0 database: *database0 collectionName: &collection0Name coll0 + - client: + id: &client1 client1 + observeEvents: + - commandStartedEvent - database: id: &database1 database1 client: *client0 @@ -33,6 +37,14 @@ createEntities: id: &collection1 collection1 database: *database0 collectionName: &collection1Name datakeys + - database: + id: &database2 database2 + client: *client1 + databaseName: &database0Name crud-tests + - collection: + id: &collection2 collection2 + database: *database2 + collectionName: &collection0Name coll0 initialData: @@ -87,13 +99,14 @@ tests: expectResult: - _id: 1 encryptedInt: 44 - outcome: - - collectionName: *collection0Name - databaseName: *database0Name - documents: + - object: *collection2 + name: find + arguments: + filter: {} + expectResult: - { _id: 1, encryptedInt: { $$type: binData }, __safeContent__: { $$type: array} } - - { _id: 2, encryptedInt: { $$type: binData }, __safeContent__: { $$type: array} } - - { _id: 3, encryptedInt: { $$type: binData }, __safeContent__: { $$type: array} } + - { _id: 2, encryptedInt: { $$type: binData }, __safeContent__: { $$type: array} } + - { _id: 3, encryptedInt: { $$type: binData }, __safeContent__: { $$type: array} } - description: client bulkWrite QE with multiple replace fails operations: - object: *client0 From 28b2ced08b832f7553517b525cad8ef40898de81 Mon Sep 17 00:00:00 2001 From: mdb-ad <198671546+mdb-ad@users.noreply.github.com> Date: Wed, 23 Apr 2025 20:19:53 -0700 Subject: [PATCH 14/33] remove bulkWrite QE error --- source/crud/bulk-write.md | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/source/crud/bulk-write.md b/source/crud/bulk-write.md index 285a5bb338..cab2c49509 100644 --- a/source/crud/bulk-write.md +++ b/source/crud/bulk-write.md @@ -634,13 +634,6 @@ write concern containing the following message: > Cannot request unacknowledged write concern and ordered writes -## Auto-Encryption - -If `MongoClient.bulkWrite` is called on a `MongoClient` configured with `AutoEncryptionOpts`, drivers MUST return an -error with the message: "bulkWrite does not currently support automatic encryption". - -This is expected to be removed once [DRIVERS-2888](https://jira.mongodb.org/browse/DRIVERS-2888) is implemented. - ## Command Batching Drivers MUST accept an arbitrary number of operations as input to the `MongoClient.bulkWrite` method. Because the server @@ -917,6 +910,8 @@ error in this specific situation does not seem helpful enough to require size ch ## **Changelog** +- 2025-04-23: Removed the requirement to error when QE is enabled. + - 2024-11-05: Updated the requirements regarding the size validation. - 2024-10-07: Error if `w:0` is used with `ordered=true` or `verboseResults=true`. From 6415c1a3c555fa18345ba95248a801ef2dd6d927 Mon Sep 17 00:00:00 2001 From: mdb-ad <198671546+mdb-ad@users.noreply.github.com> Date: Wed, 16 Jul 2025 22:15:02 -0700 Subject: [PATCH 15/33] add csfle requirement to test --- source/crud/tests/unified/client-bulkWrite-qe.json | 3 ++- source/crud/tests/unified/client-bulkWrite-qe.yml | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/source/crud/tests/unified/client-bulkWrite-qe.json b/source/crud/tests/unified/client-bulkWrite-qe.json index dad3f3950a..f2e0c0929a 100644 --- a/source/crud/tests/unified/client-bulkWrite-qe.json +++ b/source/crud/tests/unified/client-bulkWrite-qe.json @@ -4,7 +4,8 @@ "runOnRequirements": [ { "minServerVersion": "8.0", - "serverless": "forbid" + "serverless": "forbid", + "csfle": true } ], "createEntities": [ diff --git a/source/crud/tests/unified/client-bulkWrite-qe.yml b/source/crud/tests/unified/client-bulkWrite-qe.yml index 3c29884932..40564e807f 100644 --- a/source/crud/tests/unified/client-bulkWrite-qe.yml +++ b/source/crud/tests/unified/client-bulkWrite-qe.yml @@ -5,6 +5,7 @@ schemaVersion: "1.23" runOnRequirements: - minServerVersion: "8.0" serverless: forbid # Serverless does not support bulkWrite: CLOUDP-256344. + csfle: true createEntities: - client: From a388a7ebabc82277bdd196c2778e0dc612e9fbb7 Mon Sep 17 00:00:00 2001 From: mdb-ad <198671546+mdb-ad@users.noreply.github.com> Date: Wed, 23 Jul 2025 16:25:20 -0700 Subject: [PATCH 16/33] bulkWrite prose tests --- source/client-side-encryption/tests/README.md | 37 +++++++++++++++---- 1 file changed, 29 insertions(+), 8 deletions(-) diff --git a/source/client-side-encryption/tests/README.md b/source/client-side-encryption/tests/README.md index 1cf98e3ea4..5e2e552c77 100644 --- a/source/client-side-encryption/tests/README.md +++ b/source/client-side-encryption/tests/README.md @@ -565,10 +565,13 @@ First, perform the setup. 2. Using `client`, drop and create the collection `db.coll` configured with the included JSON schema [limits/limits-schema.json](../limits/limits-schema.json). -3. Using `client`, drop the collection `keyvault.datakeys`. Insert the document +3. Using `client`, drop and create the collection `db.coll2` configured with the included encryptedFields + [limits/limits-encryptedFields.json](../limits/limits-encryptedFields.json). + +4. Using `client`, drop the collection `keyvault.datakeys`. Insert the document [limits/limits-key.json](../limits/limits-key.json) -4. Create a MongoClient configured with auto encryption (referred to as `client_encrypted`) +5. Create a MongoClient configured with auto encryption (referred to as `client_encrypted`) Configure with the `local` KMS provider as follows: @@ -580,19 +583,19 @@ First, perform the setup. Using `client_encrypted` perform the following operations: -1. Insert `{ "_id": "over_2mib_under_16mib", "unencrypted": }`. +1. Insert `{ "_id": "over_2mib_under_16mib", "unencrypted": }` into `coll`. Expect this to succeed since this is still under the `maxBsonObjectSize` limit. 2. Insert the document [limits/limits-doc.json](../limits/limits-doc.json) concatenated with - `{ "_id": "encryption_exceeds_2mib", "unencrypted": < the string "a" repeated (2097152 - 2000) times > }` Note: + `{ "_id": "encryption_exceeds_2mib", "unencrypted": < the string "a" repeated (2097152 - 2000) times > }` into `coll`. Note: limits-doc.json is a 1005 byte BSON document that encrypts to a ~10,000 byte document. Expect this to succeed since after encryption this still is below the normal maximum BSON document size. Note, before auto encryption this document is under the 2 MiB limit. After encryption it exceeds the 2 MiB limit, but does NOT exceed the 16 MiB limit. -3. Bulk insert the following: +3. Use MongoCollection.bulkWrite to insert the following into `coll`: - `{ "_id": "over_2mib_1", "unencrypted": }` - `{ "_id": "over_2mib_2", "unencrypted": }` @@ -600,7 +603,7 @@ Using `client_encrypted` perform the following operations: Expect the bulk write to succeed and split after first doc (i.e. two inserts occur). This may be verified using [command monitoring](../../command-logging-and-monitoring/command-logging-and-monitoring.md). -4. Bulk insert the following: +4. Use MongoCollection.bulkWrite insert the following into `coll`: - The document [limits/limits-doc.json](../limits/limits-doc.json) concatenated with `{ "_id": "encryption_exceeds_2mib_1", "unencrypted": < the string "a" repeated (2097152 - 2000) times > }` @@ -610,15 +613,33 @@ Using `client_encrypted` perform the following operations: Expect the bulk write to succeed and split after first doc (i.e. two inserts occur). This may be verified using [command logging and monitoring](../../command-logging-and-monitoring/command-logging-and-monitoring.md). -5. Insert `{ "_id": "under_16mib", "unencrypted": `. +5. Insert `{ "_id": "under_16mib", "unencrypted": ` into `coll`. Expect this to succeed since this is still (just) under the `maxBsonObjectSize` limit. 6. Insert the document [limits/limits-doc.json](../limits/limits-doc.json) concatenated with - `{ "_id": "encryption_exceeds_16mib", "unencrypted": < the string "a" repeated (16777216 - 2000) times > }` + `{ "_id": "encryption_exceeds_16mib", "unencrypted": < the string "a" repeated (16777216 - 2000) times > }` into `coll`. Expect this to fail since encryption results in a document exceeding the `maxBsonObjectSize` limit. +7. Use MongoClient.bulkWrite to insert the following into `coll2`: + + - `{ "_id": "over_2mib_3", "unencrypted": }` + - `{ "_id": "over_2mib_4", "unencrypted": }` + + Expect the bulk write to succeed and split after first doc (i.e. two inserts occur). This may be verified using + [command logging and monitoring](../../command-logging-and-monitoring/command-logging-and-monitoring.md). + +8. Use MongoClient.bulkWrite to insert the following into `coll2`: + + - The document [limits/limits-qe-doc.json](../limits/limits-qe-doc.json) concatenated with + `{ "_id": "encryption_exceeds_2mib_3", "foo": < the string "a" repeated (2097152 - 2000 - 1500) times > }` + - The document [limits/limits-qe-doc.json](../limits/limits-qe-doc.json) concatenated with + `{ "_id": "encryption_exceeds_2mib_4", "foo": < the string "a" repeated (2097152 - 2000 - 1500) times > }` + + Expect the bulk write to succeed and split after first doc (i.e. two inserts occur). This may be verified using + [command logging and monitoring](../../command-logging-and-monitoring/command-logging-and-monitoring.md). + Optionally, if it is possible to mock the maxWriteBatchSize (i.e. the maximum number of documents in a batch) test that setting maxWriteBatchSize=1 and inserting the two documents `{ "_id": "a" }, { "_id": "b" }` with `client_encrypted` splits the operation into two inserts. From 148ed07b86ac1db8159474a23070472d661505e6 Mon Sep 17 00:00:00 2001 From: mdb-ad <198671546+mdb-ad@users.noreply.github.com> Date: Wed, 23 Jul 2025 16:29:15 -0700 Subject: [PATCH 17/33] errorContains in test --- source/crud/tests/unified/client-bulkWrite-qe.json | 3 ++- source/crud/tests/unified/client-bulkWrite-qe.yml | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/source/crud/tests/unified/client-bulkWrite-qe.json b/source/crud/tests/unified/client-bulkWrite-qe.json index f2e0c0929a..b732793400 100644 --- a/source/crud/tests/unified/client-bulkWrite-qe.json +++ b/source/crud/tests/unified/client-bulkWrite-qe.json @@ -288,7 +288,8 @@ ] }, "expectError": { - "isError": true + "isError": true, + "errorContains": "Only insert is supported in BulkWrite with multiple operations and Queryable Encryption" } } ] diff --git a/source/crud/tests/unified/client-bulkWrite-qe.yml b/source/crud/tests/unified/client-bulkWrite-qe.yml index 40564e807f..5ee0fbe0a2 100644 --- a/source/crud/tests/unified/client-bulkWrite-qe.yml +++ b/source/crud/tests/unified/client-bulkWrite-qe.yml @@ -125,3 +125,4 @@ tests: expectError: # Expect error from mongocryptd or crypt_shared isError: true + errorContains: "Only insert is supported in BulkWrite with multiple operations and Queryable Encryption" From 10d34d7eb217404711858fa506a55cf85ccb3a93 Mon Sep 17 00:00:00 2001 From: mdb-ad <198671546+mdb-ad@users.noreply.github.com> Date: Wed, 6 Aug 2025 00:12:52 -0700 Subject: [PATCH 18/33] tests draft --- source/client-side-encryption/tests/README.md | 107 ++++++++++++++++++ 1 file changed, 107 insertions(+) diff --git a/source/client-side-encryption/tests/README.md b/source/client-side-encryption/tests/README.md index b56160d62f..6642e5c02f 100644 --- a/source/client-side-encryption/tests/README.md +++ b/source/client-side-encryption/tests/README.md @@ -3764,3 +3764,110 @@ class AutoEncryptionOpts { ``` Assert that an error is thrown. + +### 27. Text Explicit Encryption + +The Text Explicit Encryption tests utilize Queryable Encryption (QE) range protocol V2 and require MongoDB server +8.2.0+ and libmongocrypt 1.15.0+. The tests must not run against a standalone. + +Before running each of the following test cases, perform the following Test Setup. + +#### Test Setup + +Load the file `encryptedFields-text.json` as `encryptedFields`. + +Load the file +[key1-document.json](https://github.com/mongodb/specifications/tree/master/source/client-side-encryption/etc/data/keys/key1-document.json) +as `key1Document`. + +Read the `"_id"` field of `key1Document` as `key1ID`. + +Drop and create the collection `db.explicit_encryption` using `encryptedFields` as an option. See +[FLE 2 CreateCollection() and Collection.Drop()](../client-side-encryption.md#create-collection-helper). + +Drop and create the collection `keyvault.datakeys`. + +Insert `key1Document` in `keyvault.datakeys` with majority write concern. + +Create a MongoClient named `keyVaultClient`. + +Create a ClientEncryption object named `clientEncryption` with these options: + +```typescript +class ClientEncryptionOpts { + keyVaultClient: , + keyVaultNamespace: "keyvault.datakeys", + kmsProviders: { "local": { "key": } }, +} +``` + +Create a MongoClient named `encryptedClient` with these `AutoEncryptionOpts`: + +```typescript +class AutoEncryptionOpts { + keyVaultNamespace: "keyvault.datakeys", + kmsProviders: { "local": { "key": } }, + bypassQueryAnalysis: true, +} +``` + +The remaining tasks require setting `TextOpts`. [Test Setup: TextOpts](#test-setup-textopts) lists the values to use +for `RangeOpts` for each of the supported data types. + +#### Test Setup: TextOpts + +This section lists the values to use for `TextOpts` for each query type. + +1. Prefix + + ```typescript + class PrefixOpts { + strMaxQueryLength: 3, + strMinQueryLength: 1, + } + ``` + +2. Suffix + + ```typescript + class SuffixOpts { + strMaxQueryLength: 3, + strMinQueryLength: 1, + } + ``` + +3. Substring + + ```typescript + class SubstringOpts { + strMaxLength: 10, + strMaxQueryLength: 3, + strMinQueryLength: 1, + } + ``` + +Use `clientEncryption` to encrypt the string "foobarbaz". Ensure the type matches that of the encrypted field. +For example, if the encrypted field is `encryptedDoubleNoPrecision` encrypt the value 6.0. + +Encrypt using the following `EncryptOpts`: + +```typescript +class EncryptOpts { + keyId : , + algorithm: "TextPreview", + contentionFactor: 0, + textOpts: TextOpts { + caseSensitive: true, + diacriticSensitive: true, + prefix: , + suffix: + }, +} +``` + +Use `encryptedClient` to insert the following document into `db.explicit_encryption`: + +```javascript +{ "_id": 0, "encryptedText": } +``` + From f85e42ef1ef71000fa48ef66489ec62c16818987 Mon Sep 17 00:00:00 2001 From: mdb-ad <198671546+mdb-ad@users.noreply.github.com> Date: Thu, 7 Aug 2025 12:33:05 -0700 Subject: [PATCH 19/33] add limits-qe-doc --- source/client-side-encryption/limits/limits-qe-doc.json | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 source/client-side-encryption/limits/limits-qe-doc.json diff --git a/source/client-side-encryption/limits/limits-qe-doc.json b/source/client-side-encryption/limits/limits-qe-doc.json new file mode 100644 index 0000000000..71efbf4068 --- /dev/null +++ b/source/client-side-encryption/limits/limits-qe-doc.json @@ -0,0 +1,3 @@ +{ + "foo": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" +} \ No newline at end of file From 6a0daee060f086688c6130c05de4e267bc4316d7 Mon Sep 17 00:00:00 2001 From: mdb-ad <198671546+mdb-ad@users.noreply.github.com> Date: Wed, 13 Aug 2025 22:17:33 -0700 Subject: [PATCH 20/33] note server 8.0 requirement --- source/client-side-encryption/tests/README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/client-side-encryption/tests/README.md b/source/client-side-encryption/tests/README.md index 5e2e552c77..2aa3f33ae9 100644 --- a/source/client-side-encryption/tests/README.md +++ b/source/client-side-encryption/tests/README.md @@ -622,6 +622,9 @@ Using `client_encrypted` perform the following operations: Expect this to fail since encryption results in a document exceeding the `maxBsonObjectSize` limit. +> [!NOTE] +> MongoDB 8.0+ is required for MongoClient.bulkWrite + 7. Use MongoClient.bulkWrite to insert the following into `coll2`: - `{ "_id": "over_2mib_3", "unencrypted": }` From 90b37018a7d4e4019a1e1886e39309be4e825e1f Mon Sep 17 00:00:00 2001 From: mdb-ad <198671546+mdb-ad@users.noreply.github.com> Date: Wed, 13 Aug 2025 22:18:06 -0700 Subject: [PATCH 21/33] add limits-encryptedFields --- .../limits/limits-encryptedFields.json | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 source/client-side-encryption/limits/limits-encryptedFields.json diff --git a/source/client-side-encryption/limits/limits-encryptedFields.json b/source/client-side-encryption/limits/limits-encryptedFields.json new file mode 100644 index 0000000000..c52a0271e1 --- /dev/null +++ b/source/client-side-encryption/limits/limits-encryptedFields.json @@ -0,0 +1,14 @@ +{ + "fields": [ + { + "keyId": { + "$binary": { + "base64": "LOCALAAAAAAAAAAAAAAAAA==", + "subType": "04" + } + }, + "path": "foo", + "bsonType": "string" + } + ] +} \ No newline at end of file From c046c988074c86adf4a923602b16a5abfe28e7b9 Mon Sep 17 00:00:00 2001 From: Isabel Atkinson Date: Mon, 11 Aug 2025 09:25:29 -0600 Subject: [PATCH 22/33] add language for auto-encryption to bulk write spec --- source/crud/bulk-write.md | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/source/crud/bulk-write.md b/source/crud/bulk-write.md index cab2c49509..2972376c42 100644 --- a/source/crud/bulk-write.md +++ b/source/crud/bulk-write.md @@ -449,7 +449,7 @@ class BulkWriteResult { * The results of each individual write operation that was successfully performed. * * This value will only be populated if the verboseResults option was set to true. - */ + */ verboseResults: Optional; /* rest of fields */ @@ -542,7 +542,9 @@ The `bulkWrite` server command has the following format: } ``` -Drivers MUST use document sequences ([`OP_MSG`](../message/OP_MSG.md) payload type 1) for the `ops` and `nsInfo` fields. +If auto-encryption is not enabled, drivers MUST use document sequences ([`OP_MSG`](../message/OP_MSG.md) payload type 1) for +the `ops` and `nsInfo` fields. If auto-encryption is enabled, drivers MUST NOT use document sequences and MUST append the +`ops` and `nsInfo` fields to the `bulkWrite` command document. The `bulkWrite` command is executed on the "admin" database. @@ -654,8 +656,10 @@ multiple commands if the user provides more than `maxWriteBatchSize` operations ### Total Message Size -Drivers MUST ensure that the total size of the `OP_MSG` built for each `bulkWrite` command does not exceed -`maxMessageSizeBytes`. +#### Unencrypted bulk writes + +When auto-encryption is not enabled, drivers MUST ensure that the total size of the `OP_MSG` built for each `bulkWrite` +command does not exceed `maxMessageSizeBytes`. The upper bound for the size of an `OP_MSG` includes opcode-related bytes (e.g. the `OP_MSG` header) and operation-agnostic command field bytes (e.g. `txnNumber`, `lsid`). Drivers MUST limit the combined size of the @@ -709,6 +713,12 @@ was determined. Drivers MUST return an error if there is not room to add at least one operation to `ops`. +#### Auto-encrypted bulk writes + +Drivers MUST use the reduced size limit defined in +[Size limits for Write Commands](../client-side-encryption/client-side-encryption.md#size-limits-for-write-commands) +for the size of the `bulkWrite` command document when auto-encryption is enabled. + ## Handling the `bulkWrite` Server Response The server's response to `bulkWrite` has the following format: @@ -839,6 +849,12 @@ When a `getMore` fails with a retryable error when attempting to iterate the res entire `bulkWrite` command to receive a fresh cursor and retry iteration. This work was omitted to minimize the scope of the initial implementation and testing of the new bulk write API, but may be revisited in the future. +### Use document sequences for auto-encrypted bulk writes + +Auto-encryption does not currently support document sequences. This specification should be updated when +[DRIVERS-2859](https://jira.mongodb.org/browse/DRIVERS-2859) is completed to require use of document sequences for `ops` +and `nsInfo` when auto-encryption is enabled. + ## Q&A ### Is `bulkWrite` supported on Atlas Serverless? From 0f4e11e78f50cf7f4dd167d5ba37a519dd8164f5 Mon Sep 17 00:00:00 2001 From: mdb-ad <198671546+mdb-ad@users.noreply.github.com> Date: Thu, 14 Aug 2025 00:49:57 -0700 Subject: [PATCH 23/33] format --- source/crud/bulk-write.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/source/crud/bulk-write.md b/source/crud/bulk-write.md index 3f5eea6718..751fc3e4ad 100644 --- a/source/crud/bulk-write.md +++ b/source/crud/bulk-write.md @@ -553,9 +553,9 @@ The `bulkWrite` server command has the following format: } ``` -If auto-encryption is not enabled, drivers MUST use document sequences ([`OP_MSG`](../message/OP_MSG.md) payload type 1) for -the `ops` and `nsInfo` fields. If auto-encryption is enabled, drivers MUST NOT use document sequences and MUST append the -`ops` and `nsInfo` fields to the `bulkWrite` command document. +If auto-encryption is not enabled, drivers MUST use document sequences ([`OP_MSG`](../message/OP_MSG.md) payload type 1) +for the `ops` and `nsInfo` fields. If auto-encryption is enabled, drivers MUST NOT use document sequences and MUST +append the `ops` and `nsInfo` fields to the `bulkWrite` command document. The `bulkWrite` command is executed on the "admin" database. @@ -727,8 +727,8 @@ Drivers MUST return an error if there is not room to add at least one operation #### Auto-encrypted bulk writes Drivers MUST use the reduced size limit defined in -[Size limits for Write Commands](../client-side-encryption/client-side-encryption.md#size-limits-for-write-commands) -for the size of the `bulkWrite` command document when auto-encryption is enabled. +[Size limits for Write Commands](../client-side-encryption/client-side-encryption.md#size-limits-for-write-commands) for +the size of the `bulkWrite` command document when auto-encryption is enabled. ## Handling the `bulkWrite` Server Response From 989e75ddb177a990e13ed231b9e7361e6c8594da Mon Sep 17 00:00:00 2001 From: mdb-ad <198671546+mdb-ad@users.noreply.github.com> Date: Wed, 20 Aug 2025 22:46:59 -0700 Subject: [PATCH 24/33] Apply suggestions from code review Co-authored-by: Kevin Albertson --- source/client-side-encryption/tests/README.md | 2 +- source/crud/bulk-write.md | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/source/client-side-encryption/tests/README.md b/source/client-side-encryption/tests/README.md index 7768b03cc0..14dfeca0ff 100644 --- a/source/client-side-encryption/tests/README.md +++ b/source/client-side-encryption/tests/README.md @@ -563,7 +563,7 @@ First, perform the setup. 2. Using `client`, drop and create the collection `db.coll` configured with the included JSON schema [limits/limits-schema.json](../limits/limits-schema.json). -3. Using `client`, drop and create the collection `db.coll2` configured with the included encryptedFields +3. If using MongoDB 8.0+, use `client` to drop and create the collection `db.coll2` configured with the included encryptedFields [limits/limits-encryptedFields.json](../limits/limits-encryptedFields.json). 4. Using `client`, drop the collection `keyvault.datakeys`. Insert the document diff --git a/source/crud/bulk-write.md b/source/crud/bulk-write.md index 751fc3e4ad..b07329ef22 100644 --- a/source/crud/bulk-write.md +++ b/source/crud/bulk-write.md @@ -866,6 +866,9 @@ Auto-encryption does not currently support document sequences. This specificatio [DRIVERS-2859](https://jira.mongodb.org/browse/DRIVERS-2859) is completed to require use of document sequences for `ops` and `nsInfo` when auto-encryption is enabled. +Drivers requiring significant changes to pass a bulkWrite command to libmongocrypt are recommended to wait until +[DRIVERS-2859](https://jira.mongodb.org/browse/DRIVERS-2859) is implemented before supporting automatic encryption. + ## Q&A ### Is `bulkWrite` supported on Atlas Serverless? From 27a61ce73f904da5b3c1762622d0e64137825538 Mon Sep 17 00:00:00 2001 From: Preston Vasquez Date: Tue, 19 Aug 2025 20:35:59 -0600 Subject: [PATCH 25/33] DRIVERS-3261 Remove IndexNotFound error checks for dropIndex (#1827) Co-authored-by: Bailey Pearson --- .../etc/generate-basic-tests.py | 43 ++++++------ .../tests/deprecated-options.json | 67 +++++++++++++++---- .../tests/deprecated-options.yml | 54 +++++++++++---- .../tests/global-timeoutMS.json | 20 ++++-- .../tests/global-timeoutMS.yml | 17 +++-- .../tests/override-collection-timeoutMS.json | 40 ++++++++--- .../tests/override-collection-timeoutMS.yml | 32 ++++++--- .../tests/override-database-timeoutMS.json | 40 ++++++++--- .../tests/override-database-timeoutMS.yml | 32 ++++++--- .../tests/override-operation-timeoutMS.json | 40 +++++++++-- .../tests/override-operation-timeoutMS.yml | 32 +++++++-- 11 files changed, 316 insertions(+), 101 deletions(-) diff --git a/source/client-side-operations-timeout/etc/generate-basic-tests.py b/source/client-side-operations-timeout/etc/generate-basic-tests.py index f52f749b8c..e70751c101 100644 --- a/source/client-side-operations-timeout/etc/generate-basic-tests.py +++ b/source/client-side-operations-timeout/etc/generate-basic-tests.py @@ -63,7 +63,7 @@ # operations. Individual generation functions can choose to include them if needed. OPERATIONS = CLIENT_OPERATIONS + DB_OPERATIONS + COLLECTION_OPERATIONS -RETRYABLE_WRITE_OPERATIONS = [op for op in OPERATIONS if op.operation_name in +RETRYABLE_WRITE_OPERATIONS = [op for op in OPERATIONS if op.operation_name in ['insertOne', 'updateOne', 'deleteOne', 'replaceOne', 'findOneAndDelete', 'findOneAndUpdate', 'findOneAndReplace', 'insertMany', 'bulkWrite'] ] @@ -102,28 +102,33 @@ def generate(name, operations): } write_yaml(name, template, injections) -def generate_global_timeout_tests(): - generate('global-timeoutMS', OPERATIONS) - -def generate_override_db(): - generate('override-database-timeoutMS', DB_OPERATIONS + COLLECTION_OPERATIONS) - -def generate_override_coll(): - generate('override-collection-timeoutMS', COLLECTION_OPERATIONS) - -def generate_override_operation(): - generate('override-operation-timeoutMS', OPERATIONS) +# TODO(DRIVERS-3266): Investigate dropping generator script for index-related +# timeoutMS tests +#def generate_global_timeout_tests(): +# generate('global-timeoutMS', OPERATIONS) +# +#def generate_override_db(): +# generate('override-database-timeoutMS', DB_OPERATIONS + COLLECTION_OPERATIONS) +# +#def generate_override_coll(): +# generate('override-collection-timeoutMS', COLLECTION_OPERATIONS) +# +#def generate_override_operation(): +# generate('override-operation-timeoutMS', OPERATIONS) +# +#def generate_deprecated(): +# generate('deprecated-options', OPERATIONS) def generate_retryable(): generate('retryability-timeoutMS', RETRYABLE_WRITE_OPERATIONS + RETRYABLE_READ_OPERATIONS) generate('retryability-legacy-timeouts', RETRYABLE_WRITE_OPERATIONS + RETRYABLE_READ_OPERATIONS) -def generate_deprecated(): - generate('deprecated-options', OPERATIONS) +# TODO(DRIVERS-3266): Investigate dropping generator script for index-related +# timeoutMS tests +#generate_global_timeout_tests() +#generate_override_db() +#generate_override_coll() +#generate_override_operation() +#generate_deprecated() -generate_global_timeout_tests() -generate_override_db() -generate_override_coll() -generate_override_operation() generate_retryable() -generate_deprecated() diff --git a/source/client-side-operations-timeout/tests/deprecated-options.json b/source/client-side-operations-timeout/tests/deprecated-options.json index d3e4631ff4..647e1bf792 100644 --- a/source/client-side-operations-timeout/tests/deprecated-options.json +++ b/source/client-side-operations-timeout/tests/deprecated-options.json @@ -6750,16 +6750,23 @@ } } }, + { + "name": "createIndex", + "object": "collection", + "arguments": { + "keys": { + "x": 1 + }, + "timeoutMS": 100000, + "name": "x_1" + } + }, { "name": "dropIndex", "object": "collection", "arguments": { "timeoutMS": 100000, "name": "x_1" - }, - "expectError": { - "isClientError": false, - "isTimeoutError": false } } ] @@ -6815,16 +6822,23 @@ ] } }, + { + "name": "createIndex", + "object": "collection", + "arguments": { + "keys": { + "x": 1 + }, + "timeoutMS": 100000, + "name": "x_1" + } + }, { "name": "dropIndex", "object": "collection", "arguments": { "timeoutMS": 100000, "name": "x_1" - }, - "expectError": { - "isClientError": false, - "isTimeoutError": false } } ], @@ -6832,6 +6846,12 @@ { "client": "client", "events": [ + { + "commandStartedEvent": { + "commandName": "createIndexes", + "databaseName": "test" + } + }, { "commandStartedEvent": { "commandName": "dropIndexes", @@ -6903,6 +6923,16 @@ ] } }, + { + "name": "createIndex", + "object": "collection", + "arguments": { + "keys": { + "x": 1 + }, + "name": "x_1" + } + }, { "name": "dropIndex", "object": "collection", @@ -6910,10 +6940,6 @@ "timeoutMS": 1000, "maxTimeMS": 5000, "name": "x_1" - }, - "expectError": { - "isClientError": false, - "isTimeoutError": false } } ], @@ -6921,6 +6947,12 @@ { "client": "client", "events": [ + { + "commandStartedEvent": { + "commandName": "createIndexes", + "databaseName": "test" + } + }, { "commandStartedEvent": { "commandName": "dropIndexes", @@ -7003,6 +7035,17 @@ } } }, + { + "name": "createIndex", + "object": "collection", + "arguments": { + "keys": { + "x": 1 + }, + "name": "x_1", + "timeoutMS": 100000 + } + }, { "name": "dropIndexes", "object": "collection", diff --git a/source/client-side-operations-timeout/tests/deprecated-options.yml b/source/client-side-operations-timeout/tests/deprecated-options.yml index 582a8983ae..e3378d5fa8 100644 --- a/source/client-side-operations-timeout/tests/deprecated-options.yml +++ b/source/client-side-operations-timeout/tests/deprecated-options.yml @@ -3743,16 +3743,19 @@ tests: failCommands: ["dropIndexes"] blockConnection: true blockTimeMS: 5 + # Create the index first so dropIndex doesn't return IndexNotFound on + # servers prior to 8.3. + - name: createIndex + object: *collection + arguments: + keys: { x: 1 } + timeoutMS: 100000 + name: "x_1" - name: dropIndex object: *collection arguments: timeoutMS: 100000 name: "x_1" - - expectError: - isClientError: false - isTimeoutError: false - - description: "wTimeoutMS is ignored if timeoutMS is set - dropIndex on collection" operations: - name: createEntities @@ -3781,18 +3784,26 @@ tests: - session: id: &session session client: *client + # Create the index first so dropIndex doesn't return IndexNotFound on + # servers prior to 8.3. + - name: createIndex + object: *collection + arguments: + keys: { x: 1 } + timeoutMS: 100000 + name: "x_1" - name: dropIndex object: *collection arguments: timeoutMS: 100000 name: "x_1" - - expectError: - isClientError: false - isTimeoutError: false expectEvents: - client: *client events: + - commandStartedEvent: + commandName: createIndexes + databaseName: *databaseName + - commandStartedEvent: commandName: dropIndexes databaseName: *databaseName @@ -3828,19 +3839,26 @@ tests: - session: id: &session session client: *client + # Create the index first so dropIndex doesn't return IndexNotFound on + # servers prior to 8.3. + - name: createIndex + object: *collection + arguments: + keys: { x: 1 } + name: "x_1" - name: dropIndex object: *collection arguments: timeoutMS: &timeoutMS 1000 maxTimeMS: 5000 name: "x_1" - - expectError: - isClientError: false - isTimeoutError: false expectEvents: - client: *client events: + - commandStartedEvent: + commandName: createIndexes + databaseName: *databaseName + - commandStartedEvent: commandName: dropIndexes databaseName: *databaseName @@ -3884,6 +3902,14 @@ tests: failCommands: ["dropIndexes"] blockConnection: true blockTimeMS: 5 + # Create the index first so dropIndex doesn't return IndexNotFound on + # servers prior to 8.3. + - name: createIndex + object: *collection + arguments: + keys: { x: 1 } + name: "x_1" + timeoutMS: 100000 - name: dropIndexes object: *collection arguments: @@ -3979,4 +4005,4 @@ tests: command: dropIndexes: *collectionName maxTimeMS: { $$lte: *timeoutMS } - \ No newline at end of file + diff --git a/source/client-side-operations-timeout/tests/global-timeoutMS.json b/source/client-side-operations-timeout/tests/global-timeoutMS.json index 740bbad2e2..f1edbe68e3 100644 --- a/source/client-side-operations-timeout/tests/global-timeoutMS.json +++ b/source/client-side-operations-timeout/tests/global-timeoutMS.json @@ -5621,15 +5621,21 @@ } } }, + { + "name": "createIndex", + "object": "collection", + "arguments": { + "keys": { + "x": 1 + }, + "name": "x_1" + } + }, { "name": "dropIndex", "object": "collection", "arguments": { "name": "x_1" - }, - "expectError": { - "isClientError": false, - "isTimeoutError": false } } ], @@ -5637,6 +5643,12 @@ { "client": "client", "events": [ + { + "commandStartedEvent": { + "commandName": "createIndexes", + "databaseName": "test" + } + }, { "commandStartedEvent": { "commandName": "dropIndexes", diff --git a/source/client-side-operations-timeout/tests/global-timeoutMS.yml b/source/client-side-operations-timeout/tests/global-timeoutMS.yml index 7b4a78ac78..847c26f096 100644 --- a/source/client-side-operations-timeout/tests/global-timeoutMS.yml +++ b/source/client-side-operations-timeout/tests/global-timeoutMS.yml @@ -3120,17 +3120,24 @@ tests: failCommands: ["dropIndexes"] blockConnection: true blockTimeMS: 15 + # Create the index first so dropIndex doesn't return IndexNotFound on + # servers prior to 8.3. + - name: createIndex + object: *collection + arguments: + keys: { x: 1 } + name: "x_1" - name: dropIndex object: *collection arguments: name: "x_1" - - expectError: - isClientError: false - isTimeoutError: false expectEvents: - client: *client events: + - commandStartedEvent: + commandName: createIndexes + databaseName: *databaseName + - commandStartedEvent: commandName: dropIndexes databaseName: *databaseName @@ -3233,4 +3240,4 @@ tests: command: dropIndexes: *collectionName maxTimeMS: { $$exists: false } - \ No newline at end of file + diff --git a/source/client-side-operations-timeout/tests/override-collection-timeoutMS.json b/source/client-side-operations-timeout/tests/override-collection-timeoutMS.json index d17e22fc2f..c56d45bcbb 100644 --- a/source/client-side-operations-timeout/tests/override-collection-timeoutMS.json +++ b/source/client-side-operations-timeout/tests/override-collection-timeoutMS.json @@ -3251,15 +3251,21 @@ } } }, + { + "name": "createIndex", + "object": "collection", + "arguments": { + "keys": { + "x": 1 + }, + "name": "x_1" + } + }, { "name": "dropIndex", "object": "collection", "arguments": { "name": "x_1" - }, - "expectError": { - "isClientError": false, - "isTimeoutError": false } } ], @@ -3267,6 +3273,12 @@ { "client": "client", "events": [ + { + "commandStartedEvent": { + "commandName": "createIndexes", + "databaseName": "test" + } + }, { "commandStartedEvent": { "commandName": "dropIndexes", @@ -3327,15 +3339,21 @@ } } }, + { + "name": "createIndex", + "object": "collection", + "arguments": { + "keys": { + "x": 1 + }, + "name": "x_1" + } + }, { "name": "dropIndex", "object": "collection", "arguments": { "name": "x_1" - }, - "expectError": { - "isClientError": false, - "isTimeoutError": false } } ], @@ -3343,6 +3361,12 @@ { "client": "client", "events": [ + { + "commandStartedEvent": { + "commandName": "createIndexes", + "databaseName": "test" + } + }, { "commandStartedEvent": { "commandName": "dropIndexes", diff --git a/source/client-side-operations-timeout/tests/override-collection-timeoutMS.yml b/source/client-side-operations-timeout/tests/override-collection-timeoutMS.yml index d1d1c61056..07e2a14429 100644 --- a/source/client-side-operations-timeout/tests/override-collection-timeoutMS.yml +++ b/source/client-side-operations-timeout/tests/override-collection-timeoutMS.yml @@ -1745,17 +1745,24 @@ tests: failCommands: ["dropIndexes"] blockConnection: true blockTimeMS: 15 + # Create the index first so dropIndex doesn't return IndexNotFound on + # servers prior to 8.3. + - name: createIndex + object: *collection + arguments: + keys: { x: 1 } + name: "x_1" - name: dropIndex object: *collection arguments: name: "x_1" - - expectError: - isClientError: false - isTimeoutError: false expectEvents: - client: *client events: + - commandStartedEvent: + commandName: createIndexes + databaseName: *databaseName + - commandStartedEvent: commandName: dropIndexes databaseName: *databaseName @@ -1785,17 +1792,24 @@ tests: failCommands: ["dropIndexes"] blockConnection: true blockTimeMS: 15 + # Create the index first so dropIndex doesn't return IndexNotFound on + # servers prior to 8.3. + - name: createIndex + object: *collection + arguments: + keys: { x: 1 } + name: "x_1" - name: dropIndex object: *collection arguments: name: "x_1" - - expectError: - isClientError: false - isTimeoutError: false expectEvents: - client: *client events: + - commandStartedEvent: + commandName: createIndexes + databaseName: *databaseName + - commandStartedEvent: commandName: dropIndexes databaseName: *databaseName @@ -1874,4 +1888,4 @@ tests: command: dropIndexes: *collectionName maxTimeMS: { $$exists: false } - \ No newline at end of file + diff --git a/source/client-side-operations-timeout/tests/override-database-timeoutMS.json b/source/client-side-operations-timeout/tests/override-database-timeoutMS.json index f7fa642c58..11ff7a59fd 100644 --- a/source/client-side-operations-timeout/tests/override-database-timeoutMS.json +++ b/source/client-side-operations-timeout/tests/override-database-timeoutMS.json @@ -4354,15 +4354,21 @@ } } }, + { + "name": "createIndex", + "object": "collection", + "arguments": { + "keys": { + "x": 1 + }, + "name": "x_1" + } + }, { "name": "dropIndex", "object": "collection", "arguments": { "name": "x_1" - }, - "expectError": { - "isClientError": false, - "isTimeoutError": false } } ], @@ -4370,6 +4376,12 @@ { "client": "client", "events": [ + { + "commandStartedEvent": { + "commandName": "createIndexes", + "databaseName": "test" + } + }, { "commandStartedEvent": { "commandName": "dropIndexes", @@ -4437,15 +4449,21 @@ } } }, + { + "name": "createIndex", + "object": "collection", + "arguments": { + "keys": { + "x": 1 + }, + "name": "x_1" + } + }, { "name": "dropIndex", "object": "collection", "arguments": { "name": "x_1" - }, - "expectError": { - "isClientError": false, - "isTimeoutError": false } } ], @@ -4453,6 +4471,12 @@ { "client": "client", "events": [ + { + "commandStartedEvent": { + "commandName": "createIndexes", + "databaseName": "test" + } + }, { "commandStartedEvent": { "commandName": "dropIndexes", diff --git a/source/client-side-operations-timeout/tests/override-database-timeoutMS.yml b/source/client-side-operations-timeout/tests/override-database-timeoutMS.yml index aed7b43720..4ded17164d 100644 --- a/source/client-side-operations-timeout/tests/override-database-timeoutMS.yml +++ b/source/client-side-operations-timeout/tests/override-database-timeoutMS.yml @@ -2343,17 +2343,24 @@ tests: failCommands: ["dropIndexes"] blockConnection: true blockTimeMS: 15 + # Create the index first so dropIndex doesn't return IndexNotFound on + # servers prior to 8.3. + - name: createIndex + object: *collection + arguments: + keys: { x: 1 } + name: "x_1" - name: dropIndex object: *collection arguments: name: "x_1" - - expectError: - isClientError: false - isTimeoutError: false expectEvents: - client: *client events: + - commandStartedEvent: + commandName: createIndexes + databaseName: *databaseName + - commandStartedEvent: commandName: dropIndexes databaseName: *databaseName @@ -2387,17 +2394,24 @@ tests: failCommands: ["dropIndexes"] blockConnection: true blockTimeMS: 15 + # Create the index first so dropIndex doesn't return IndexNotFound on + # servers prior to 8.3. + - name: createIndex + object: *collection + arguments: + keys: { x: 1 } + name: "x_1" - name: dropIndex object: *collection arguments: name: "x_1" - - expectError: - isClientError: false - isTimeoutError: false expectEvents: - client: *client events: + - commandStartedEvent: + commandName: createIndexes + databaseName: *databaseName + - commandStartedEvent: commandName: dropIndexes databaseName: *databaseName @@ -2484,4 +2498,4 @@ tests: command: dropIndexes: *collectionName maxTimeMS: { $$exists: false } - \ No newline at end of file + diff --git a/source/client-side-operations-timeout/tests/override-operation-timeoutMS.json b/source/client-side-operations-timeout/tests/override-operation-timeoutMS.json index 6fa0bd802a..f33f876137 100644 --- a/source/client-side-operations-timeout/tests/override-operation-timeoutMS.json +++ b/source/client-side-operations-timeout/tests/override-operation-timeoutMS.json @@ -3378,15 +3378,23 @@ } } }, + { + "name": "createIndex", + "object": "collection", + "arguments": { + "keys": { + "x": 1 + }, + "timeoutMS": 1000, + "name": "x_1" + } + }, { "name": "dropIndex", "object": "collection", "arguments": { "timeoutMS": 1000, "name": "x_1" - }, - "expectError": { - "isTimeoutError": false } } ], @@ -3394,6 +3402,12 @@ { "client": "client", "events": [ + { + "commandStartedEvent": { + "commandName": "createIndexes", + "databaseName": "test" + } + }, { "commandStartedEvent": { "commandName": "dropIndexes", @@ -3436,15 +3450,23 @@ } } }, + { + "name": "createIndex", + "object": "collection", + "arguments": { + "keys": { + "x": 1 + }, + "timeoutMS": 0, + "name": "x_1" + } + }, { "name": "dropIndex", "object": "collection", "arguments": { "timeoutMS": 0, "name": "x_1" - }, - "expectError": { - "isTimeoutError": false } } ], @@ -3452,6 +3474,12 @@ { "client": "client", "events": [ + { + "commandStartedEvent": { + "commandName": "createIndexes", + "databaseName": "test" + } + }, { "commandStartedEvent": { "commandName": "dropIndexes", diff --git a/source/client-side-operations-timeout/tests/override-operation-timeoutMS.yml b/source/client-side-operations-timeout/tests/override-operation-timeoutMS.yml index 28eabcb7c8..370e9393cf 100644 --- a/source/client-side-operations-timeout/tests/override-operation-timeoutMS.yml +++ b/source/client-side-operations-timeout/tests/override-operation-timeoutMS.yml @@ -1812,17 +1812,26 @@ tests: failCommands: ["dropIndexes"] blockConnection: true blockTimeMS: 15 + # Create the index first so dropIndex doesn't return IndexNotFound on + # servers prior to 8.3. + - name: createIndex + object: *collection + arguments: + keys: { x: 1 } + timeoutMS: 1000 + name: "x_1" - name: dropIndex object: *collection arguments: timeoutMS: 1000 name: "x_1" - - expectError: - isTimeoutError: false # IndexNotFound expectEvents: - client: *client events: + - commandStartedEvent: + commandName: createIndexes + databaseName: *databaseName + - commandStartedEvent: commandName: dropIndexes databaseName: *databaseName @@ -1842,17 +1851,26 @@ tests: failCommands: ["dropIndexes"] blockConnection: true blockTimeMS: 15 + # Create the index first so dropIndex doesn't return IndexNotFound on + # servers prior to 8.3. + - name: createIndex + object: *collection + arguments: + keys: { x: 1 } + timeoutMS: 0 + name: "x_1" - name: dropIndex object: *collection arguments: timeoutMS: 0 name: "x_1" - - expectError: - isTimeoutError: false # IndexNotFound expectEvents: - client: *client events: + - commandStartedEvent: + commandName: createIndexes + databaseName: *databaseName + - commandStartedEvent: commandName: dropIndexes databaseName: *databaseName @@ -1915,4 +1933,4 @@ tests: command: dropIndexes: *collectionName maxTimeMS: { $$exists: false } - \ No newline at end of file + From 6f40e949f14020c292b1ffe5a78bc3075c196b57 Mon Sep 17 00:00:00 2001 From: mdb-ad <198671546+mdb-ad@users.noreply.github.com> Date: Wed, 20 Aug 2025 22:41:46 -0700 Subject: [PATCH 26/33] minLibmongocryptVersion in tests --- source/crud/tests/unified/client-bulkWrite-qe.json | 6 ++++-- source/crud/tests/unified/client-bulkWrite-qe.yml | 5 +++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/source/crud/tests/unified/client-bulkWrite-qe.json b/source/crud/tests/unified/client-bulkWrite-qe.json index b732793400..3efec77493 100644 --- a/source/crud/tests/unified/client-bulkWrite-qe.json +++ b/source/crud/tests/unified/client-bulkWrite-qe.json @@ -1,11 +1,13 @@ { "description": "client bulkWrite with queryable encryption", - "schemaVersion": "1.23", + "schemaVersion": "1.25", "runOnRequirements": [ { "minServerVersion": "8.0", "serverless": "forbid", - "csfle": true + "csfle": { + "minLibmongocryptVersion": "1.10.0" + } } ], "createEntities": [ diff --git a/source/crud/tests/unified/client-bulkWrite-qe.yml b/source/crud/tests/unified/client-bulkWrite-qe.yml index 5ee0fbe0a2..3135c31346 100644 --- a/source/crud/tests/unified/client-bulkWrite-qe.yml +++ b/source/crud/tests/unified/client-bulkWrite-qe.yml @@ -1,11 +1,12 @@ description: client bulkWrite with queryable encryption -schemaVersion: "1.23" +schemaVersion: "1.25" runOnRequirements: - minServerVersion: "8.0" serverless: forbid # Serverless does not support bulkWrite: CLOUDP-256344. - csfle: true + csfle: + minLibmongocryptVersion: "1.10.0" createEntities: - client: From 74d3d3e31eaf2f3f3eb6ee58c6c2ad95e4bab8f9 Mon Sep 17 00:00:00 2001 From: mdb-ad <198671546+mdb-ad@users.noreply.github.com> Date: Wed, 20 Aug 2025 22:45:43 -0700 Subject: [PATCH 27/33] rephrase server version minimum --- source/client-side-encryption/tests/README.md | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/source/client-side-encryption/tests/README.md b/source/client-side-encryption/tests/README.md index 14dfeca0ff..a322bf12f3 100644 --- a/source/client-side-encryption/tests/README.md +++ b/source/client-side-encryption/tests/README.md @@ -620,10 +620,7 @@ Using `client_encrypted` perform the following operations: Expect this to fail since encryption results in a document exceeding the `maxBsonObjectSize` limit. -> [!NOTE] -> MongoDB 8.0+ is required for MongoClient.bulkWrite - -7. Use MongoClient.bulkWrite to insert the following into `coll2`: +7. If using MongoDB 8.0+, use MongoClient.bulkWrite to insert the following into `coll2`: - `{ "_id": "over_2mib_3", "unencrypted": }` - `{ "_id": "over_2mib_4", "unencrypted": }` @@ -631,7 +628,7 @@ Using `client_encrypted` perform the following operations: Expect the bulk write to succeed and split after first doc (i.e. two inserts occur). This may be verified using [command logging and monitoring](../../command-logging-and-monitoring/command-logging-and-monitoring.md). -8. Use MongoClient.bulkWrite to insert the following into `coll2`: +8. If using MongoDB 8.0+, use MongoClient.bulkWrite to insert the following into `coll2`: - The document [limits/limits-qe-doc.json](../limits/limits-qe-doc.json) concatenated with `{ "_id": "encryption_exceeds_2mib_3", "foo": < the string "a" repeated (2097152 - 2000 - 1500) times > }` From 35f636c3dcd899f7fcb9b010e5e2006376ff249d Mon Sep 17 00:00:00 2001 From: mdb-ad <198671546+mdb-ad@users.noreply.github.com> Date: Wed, 20 Aug 2025 22:46:02 -0700 Subject: [PATCH 28/33] remove prose test 13 --- source/crud/tests/README.md | 41 +++---------------------------------- 1 file changed, 3 insertions(+), 38 deletions(-) diff --git a/source/crud/tests/README.md b/source/crud/tests/README.md index 92365a6f05..b9418fe359 100644 --- a/source/crud/tests/README.md +++ b/source/crud/tests/README.md @@ -607,42 +607,7 @@ Execute `bulkWrite` on `client` with `largeNamespaceModel`. Assert that an error Assert that `error` is a client error. If a `BulkWriteException` was thrown, assert `BulkWriteException.partialResult` is unset. -### 13. `MongoClient.bulkWrite` returns an error if auto-encryption is configured - -This test is expected to be removed when [DRIVERS-2888](https://jira.mongodb.org/browse/DRIVERS-2888) is resolved. - -Test that `MongoClient.bulkWrite` returns an error if the client has auto-encryption configured. - -This test must only be run on 8.0+ servers. This test must be skipped on Atlas Serverless. - -Construct a `MongoClient` (referred to as `client`) configured with the following `AutoEncryptionOpts`: - -```javascript -AutoEncryptionOpts { - "keyVaultNamespace": "db.coll", - "kmsProviders": { - "aws": { - "accessKeyId": "foo", - "secretAccessKey": "bar" - } - } -} -``` - -Construct the following write model (referred to as `model`): - -```javascript -InsertOne { - "namespace": "db.coll", - "document": { "a": "b" } -} -``` - -Execute `bulkWrite` on `client` with `model`. Assert that an error (referred to as `error`) is returned. Assert that -`error` is a client error containing the message: "bulkWrite does not currently support automatic encryption". If a -`BulkWriteException` was thrown, assert `BulkWriteException.partialResult` is unset. - -### 14. `explain` helpers allow users to specify `maxTimeMS` +### 13. `explain` helpers allow users to specify `maxTimeMS` Drivers that provide multiple APIs to specify explain should ensure this test is run at least once with each distinct API. For example, the Node driver runs this test with option API (`collection.find({}, { explain: ... })`) and the @@ -658,7 +623,7 @@ maxTimeMS value of 2000ms for the `explain`. Obtain the command started event for the explain. Confirm that the top-level explain command should has a `maxTimeMS` value of `2000`. -### 15. `MongoClient.bulkWrite` with unacknowledged write concern uses `w:0` for all batches +### 14. `MongoClient.bulkWrite` with unacknowledged write concern uses `w:0` for all batches This test must only be run on 8.0+ servers. This test must be skipped on Atlas Serverless. @@ -700,7 +665,7 @@ To force completion of the `w:0` writes, execute `coll.countDocuments` and expec `maxMessageSizeBytes / maxBsonObjectSize + 1`. This is intended to avoid incomplete writes interfering with other tests that may use this collection. -### 16. Generated document identifiers are the first field in their document +### 15. Generated document identifiers are the first field in their document Construct a `MongoClient` (referred to as `client`) with [command monitoring](../../command-logging-and-monitoring/command-logging-and-monitoring.md) enabled to observe From 7aba088633485547476840451ef371ea19fb184b Mon Sep 17 00:00:00 2001 From: mdb-ad <198671546+mdb-ad@users.noreply.github.com> Date: Wed, 20 Aug 2025 22:51:00 -0700 Subject: [PATCH 29/33] format --- source/client-side-encryption/tests/README.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/source/client-side-encryption/tests/README.md b/source/client-side-encryption/tests/README.md index a322bf12f3..6deb4f81a3 100644 --- a/source/client-side-encryption/tests/README.md +++ b/source/client-side-encryption/tests/README.md @@ -563,8 +563,8 @@ First, perform the setup. 2. Using `client`, drop and create the collection `db.coll` configured with the included JSON schema [limits/limits-schema.json](../limits/limits-schema.json). -3. If using MongoDB 8.0+, use `client` to drop and create the collection `db.coll2` configured with the included encryptedFields - [limits/limits-encryptedFields.json](../limits/limits-encryptedFields.json). +3. If using MongoDB 8.0+, use `client` to drop and create the collection `db.coll2` configured with the included + encryptedFields [limits/limits-encryptedFields.json](../limits/limits-encryptedFields.json). 4. Using `client`, drop the collection `keyvault.datakeys`. Insert the document [limits/limits-key.json](../limits/limits-key.json) @@ -586,8 +586,8 @@ Using `client_encrypted` perform the following operations: Expect this to succeed since this is still under the `maxBsonObjectSize` limit. 2. Insert the document [limits/limits-doc.json](../limits/limits-doc.json) concatenated with - `{ "_id": "encryption_exceeds_2mib", "unencrypted": < the string "a" repeated (2097152 - 2000) times > }` into `coll`. Note: - limits-doc.json is a 1005 byte BSON document that encrypts to a ~10,000 byte document. + `{ "_id": "encryption_exceeds_2mib", "unencrypted": < the string "a" repeated (2097152 - 2000) times > }` into + `coll`. Note: limits-doc.json is a 1005 byte BSON document that encrypts to a ~10,000 byte document. Expect this to succeed since after encryption this still is below the normal maximum BSON document size. Note, before auto encryption this document is under the 2 MiB limit. After encryption it exceeds the 2 MiB limit, but does NOT @@ -616,7 +616,8 @@ Using `client_encrypted` perform the following operations: Expect this to succeed since this is still (just) under the `maxBsonObjectSize` limit. 6. Insert the document [limits/limits-doc.json](../limits/limits-doc.json) concatenated with - `{ "_id": "encryption_exceeds_16mib", "unencrypted": < the string "a" repeated (16777216 - 2000) times > }` into `coll`. + `{ "_id": "encryption_exceeds_16mib", "unencrypted": < the string "a" repeated (16777216 - 2000) times > }` into + `coll`. Expect this to fail since encryption results in a document exceeding the `maxBsonObjectSize` limit. @@ -634,7 +635,7 @@ Using `client_encrypted` perform the following operations: `{ "_id": "encryption_exceeds_2mib_3", "foo": < the string "a" repeated (2097152 - 2000 - 1500) times > }` - The document [limits/limits-qe-doc.json](../limits/limits-qe-doc.json) concatenated with `{ "_id": "encryption_exceeds_2mib_4", "foo": < the string "a" repeated (2097152 - 2000 - 1500) times > }` - + Expect the bulk write to succeed and split after first doc (i.e. two inserts occur). This may be verified using [command logging and monitoring](../../command-logging-and-monitoring/command-logging-and-monitoring.md). From 666880f07696aab9ea539dcf9abc585b041d1bd1 Mon Sep 17 00:00:00 2001 From: mdb-ad <198671546+mdb-ad@users.noreply.github.com> Date: Thu, 21 Aug 2025 11:03:40 -0700 Subject: [PATCH 30/33] restore prose test numbering Co-authored-by: Kevin Albertson --- source/crud/tests/README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/source/crud/tests/README.md b/source/crud/tests/README.md index b9418fe359..061d1512f3 100644 --- a/source/crud/tests/README.md +++ b/source/crud/tests/README.md @@ -623,7 +623,9 @@ maxTimeMS value of 2000ms for the `explain`. Obtain the command started event for the explain. Confirm that the top-level explain command should has a `maxTimeMS` value of `2000`. -### 14. `MongoClient.bulkWrite` with unacknowledged write concern uses `w:0` for all batches +### 14. *Removed* + +### 15. `MongoClient.bulkWrite` with unacknowledged write concern uses `w:0` for all batches This test must only be run on 8.0+ servers. This test must be skipped on Atlas Serverless. @@ -665,7 +667,7 @@ To force completion of the `w:0` writes, execute `coll.countDocuments` and expec `maxMessageSizeBytes / maxBsonObjectSize + 1`. This is intended to avoid incomplete writes interfering with other tests that may use this collection. -### 15. Generated document identifiers are the first field in their document +### 16. Generated document identifiers are the first field in their document Construct a `MongoClient` (referred to as `client`) with [command monitoring](../../command-logging-and-monitoring/command-logging-and-monitoring.md) enabled to observe From 8b628d6966cd29769132d89712f15b10d4c340bf Mon Sep 17 00:00:00 2001 From: mdb-ad <198671546+mdb-ad@users.noreply.github.com> Date: Thu, 21 Aug 2025 11:14:00 -0700 Subject: [PATCH 31/33] removed test was 13 --- source/crud/tests/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/crud/tests/README.md b/source/crud/tests/README.md index 061d1512f3..465fff7db6 100644 --- a/source/crud/tests/README.md +++ b/source/crud/tests/README.md @@ -607,7 +607,9 @@ Execute `bulkWrite` on `client` with `largeNamespaceModel`. Assert that an error Assert that `error` is a client error. If a `BulkWriteException` was thrown, assert `BulkWriteException.partialResult` is unset. -### 13. `explain` helpers allow users to specify `maxTimeMS` +### 13. *Removed* + +### 14. `explain` helpers allow users to specify `maxTimeMS` Drivers that provide multiple APIs to specify explain should ensure this test is run at least once with each distinct API. For example, the Node driver runs this test with option API (`collection.find({}, { explain: ... })`) and the @@ -623,8 +625,6 @@ maxTimeMS value of 2000ms for the `explain`. Obtain the command started event for the explain. Confirm that the top-level explain command should has a `maxTimeMS` value of `2000`. -### 14. *Removed* - ### 15. `MongoClient.bulkWrite` with unacknowledged write concern uses `w:0` for all batches This test must only be run on 8.0+ servers. This test must be skipped on Atlas Serverless. From 449dabdfac84b26cb3da4ec335eae4d512ff6e9d Mon Sep 17 00:00:00 2001 From: mdb-ad <198671546+mdb-ad@users.noreply.github.com> Date: Fri, 22 Aug 2025 11:48:06 -0700 Subject: [PATCH 32/33] no standalone on qe test Co-authored-by: Kevin Albertson --- source/crud/tests/unified/client-bulkWrite-qe.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/source/crud/tests/unified/client-bulkWrite-qe.yml b/source/crud/tests/unified/client-bulkWrite-qe.yml index 3135c31346..1c7adadae0 100644 --- a/source/crud/tests/unified/client-bulkWrite-qe.yml +++ b/source/crud/tests/unified/client-bulkWrite-qe.yml @@ -4,6 +4,7 @@ schemaVersion: "1.25" runOnRequirements: - minServerVersion: "8.0" + topologies: ["replicaset", "sharded", "load-balanced"] # QE does not support standalone. serverless: forbid # Serverless does not support bulkWrite: CLOUDP-256344. csfle: minLibmongocryptVersion: "1.10.0" From 22d962568ddaf9ece53e01e23739c398531e48a8 Mon Sep 17 00:00:00 2001 From: Kevin Albertson Date: Fri, 22 Aug 2025 15:08:11 -0400 Subject: [PATCH 33/33] update JSON file --- source/crud/tests/unified/client-bulkWrite-qe.json | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/source/crud/tests/unified/client-bulkWrite-qe.json b/source/crud/tests/unified/client-bulkWrite-qe.json index 3efec77493..9ed46025a6 100644 --- a/source/crud/tests/unified/client-bulkWrite-qe.json +++ b/source/crud/tests/unified/client-bulkWrite-qe.json @@ -4,6 +4,11 @@ "runOnRequirements": [ { "minServerVersion": "8.0", + "topologies": [ + "replicaset", + "sharded", + "load-balanced" + ], "serverless": "forbid", "csfle": { "minLibmongocryptVersion": "1.10.0"