diff --git a/Microsoft.Azure.Cosmos/src/CosmosClient.cs b/Microsoft.Azure.Cosmos/src/CosmosClient.cs index 574a1e1044..eb1792f81f 100644 --- a/Microsoft.Azure.Cosmos/src/CosmosClient.cs +++ b/Microsoft.Azure.Cosmos/src/CosmosClient.cs @@ -76,6 +76,10 @@ static CosmosClient() { HttpConstants.Versions.CurrentVersion = HttpConstants.Versions.v2018_12_31; HttpConstants.Versions.CurrentVersionUTF8 = Encoding.UTF8.GetBytes(HttpConstants.Versions.CurrentVersion); + + // V3 always assumes assemblies exists + // Shall revisit on feedback + ServiceInteropWrapper.AssembliesExist = new Lazy(() => true); } /// diff --git a/Microsoft.Azure.Cosmos/src/Microsoft.Azure.Cosmos.csproj b/Microsoft.Azure.Cosmos/src/Microsoft.Azure.Cosmos.csproj index 7a0c94a00d..0227acd05b 100644 --- a/Microsoft.Azure.Cosmos/src/Microsoft.Azure.Cosmos.csproj +++ b/Microsoft.Azure.Cosmos/src/Microsoft.Azure.Cosmos.csproj @@ -7,7 +7,7 @@ © Microsoft Corporation. All rights reserved. en-US 3.0.0.12-preview - 3.0.0.27-preview + 3.0.0.28-preview $(ClientVersion)-nightly$(CurrentDate) $(ClientVersion) $(VersionPrefix) @@ -61,5 +61,5 @@ $(DefineConstants);DOCDBCLIENT;NETSTANDARD20 $(DefineConstants);SignAssembly - + diff --git a/Microsoft.Azure.Cosmos/src/Query/CosmosOrderByItemQueryExecutionContext.cs b/Microsoft.Azure.Cosmos/src/Query/CosmosOrderByItemQueryExecutionContext.cs index 4798578ca7..d809398f29 100644 --- a/Microsoft.Azure.Cosmos/src/Query/CosmosOrderByItemQueryExecutionContext.cs +++ b/Microsoft.Azure.Cosmos/src/Query/CosmosOrderByItemQueryExecutionContext.cs @@ -19,6 +19,7 @@ namespace Microsoft.Azure.Cosmos.Query using Microsoft.Azure.Cosmos.Internal; using Microsoft.Azure.Documents; using Microsoft.Azure.Cosmos.CosmosElements; + using System.Net; /// /// CosmosOrderByItemQueryExecutionContext is a concrete implementation for CrossPartitionQueryExecutionContext. @@ -385,7 +386,7 @@ private OrderByContinuationToken[] ValidateAndExtractContinuationToken( if (suppliedOrderByContinuationTokens.Length == 0) { this.TraceWarning($"Order by continuation token can not be empty: {requestContinuation}."); - throw new BadRequestException(RMResources.InvalidContinuationToken); + throw new CosmosException(HttpStatusCode.BadRequest, RMResources.InvalidContinuationToken); } foreach (OrderByContinuationToken suppliedOrderByContinuationToken in suppliedOrderByContinuationTokens) @@ -393,7 +394,7 @@ private OrderByContinuationToken[] ValidateAndExtractContinuationToken( if (suppliedOrderByContinuationToken.OrderByItems.Count != sortOrders.Length) { this.TraceWarning($"Invalid order-by items in continuation token {requestContinuation} for OrderBy~Context."); - throw new BadRequestException(RMResources.InvalidContinuationToken); + throw new CosmosException(HttpStatusCode.BadRequest, RMResources.InvalidContinuationToken); } } @@ -403,7 +404,7 @@ private OrderByContinuationToken[] ValidateAndExtractContinuationToken( { this.TraceWarning($"Invalid JSON in continuation token {requestContinuation} for OrderBy~Context, exception: {ex.Message}"); - throw new BadRequestException(RMResources.InvalidContinuationToken, ex); + throw new CosmosException(HttpStatusCode.BadRequest, RMResources.InvalidContinuationToken); } } @@ -478,7 +479,7 @@ private async Task FilterAsync( CultureInfo.InvariantCulture, "Invalid Rid in the continuation token {0} for OrderBy~Context.", continuationToken.CompositeContinuationToken.Token)); - throw new BadRequestException(RMResources.InvalidContinuationToken); + throw new CosmosException(HttpStatusCode.BadRequest, RMResources.InvalidContinuationToken); } resourceIds.Add(orderByResult.Rid, rid); @@ -492,7 +493,7 @@ private async Task FilterAsync( CultureInfo.InvariantCulture, "Invalid Rid in the continuation token {0} for OrderBy~Context.", continuationToken.CompositeContinuationToken.Token)); - throw new BadRequestException(RMResources.InvalidContinuationToken); + throw new CosmosException(HttpStatusCode.BadRequest, RMResources.InvalidContinuationToken); } continuationRidVerified = true; diff --git a/Microsoft.Azure.Cosmos/src/Query/CosmosParallelItemQueryExecutionContext.cs b/Microsoft.Azure.Cosmos/src/Query/CosmosParallelItemQueryExecutionContext.cs index def4832c2d..4134f4b3c1 100644 --- a/Microsoft.Azure.Cosmos/src/Query/CosmosParallelItemQueryExecutionContext.cs +++ b/Microsoft.Azure.Cosmos/src/Query/CosmosParallelItemQueryExecutionContext.cs @@ -17,6 +17,7 @@ namespace Microsoft.Azure.Cosmos.Query using Microsoft.Azure.Cosmos.Internal; using Microsoft.Azure.Documents; using Microsoft.Azure.Cosmos.CosmosElements; + using System.Net; /// /// CosmosParallelItemQueryExecutionContext is a concrete implementation for CrossPartitionQueryExecutionContext. @@ -200,7 +201,7 @@ private async Task InitializeAsync( CultureInfo.InvariantCulture, "Invalid Range in the continuation token {0} for Parallel~Context.", requestContinuation)); - throw new BadRequestException(RMResources.InvalidContinuationToken); + throw new CosmosException(HttpStatusCode.BadRequest, RMResources.InvalidContinuationToken); } } @@ -210,7 +211,7 @@ private async Task InitializeAsync( CultureInfo.InvariantCulture, "Invalid format for continuation token {0} for Parallel~Context.", requestContinuation)); - throw new BadRequestException(RMResources.InvalidContinuationToken); + throw new CosmosException(HttpStatusCode.BadRequest, RMResources.InvalidContinuationToken); } } catch (JsonException ex) @@ -221,7 +222,7 @@ private async Task InitializeAsync( requestContinuation, ex.Message)); - throw new BadRequestException(RMResources.InvalidContinuationToken, ex); + throw new CosmosException(HttpStatusCode.BadRequest, RMResources.InvalidContinuationToken); } filteredPartitionKeyRanges = this.GetPartitionKeyRangesForContinuation(suppliedCompositeContinuationTokens, partitionKeyRanges, out targetIndicesForFullContinuation); diff --git a/Microsoft.Azure.Cosmos/src/Query/CosmosQueryContext.cs b/Microsoft.Azure.Cosmos/src/Query/CosmosQueryContext.cs index 63af06e387..8193d7d78b 100644 --- a/Microsoft.Azure.Cosmos/src/Query/CosmosQueryContext.cs +++ b/Microsoft.Azure.Cosmos/src/Query/CosmosQueryContext.cs @@ -17,7 +17,7 @@ internal class CosmosQueryContext public virtual ResourceType ResourceTypeEnum { get; } public virtual OperationType OperationTypeEnum { get; } public virtual Type ResourceType { get; } - public virtual SqlQuerySpec SqlQuerySpec { get; } + public SqlQuerySpec SqlQuerySpec { get; internal set; } public virtual QueryRequestOptions QueryRequestOptions { get; } public virtual bool IsContinuationExpected { get; } public virtual bool AllowNonValueAggregateQuery { get; } diff --git a/Microsoft.Azure.Cosmos/src/Query/CosmosQueryExecutionContextFactory.cs b/Microsoft.Azure.Cosmos/src/Query/CosmosQueryExecutionContextFactory.cs index 28d0acaba3..63a88e951b 100644 --- a/Microsoft.Azure.Cosmos/src/Query/CosmosQueryExecutionContextFactory.cs +++ b/Microsoft.Azure.Cosmos/src/Query/CosmosQueryExecutionContextFactory.cs @@ -132,7 +132,7 @@ private async Task GetContainerSettingsAsync(Cancellati { cancellationToken.ThrowIfCancellationRequested(); - CosmosContainerSettings containerSettings; + CosmosContainerSettings containerSettings = null; if (this.cosmosQueryContext.ResourceTypeEnum.IsCollectionChild()) { CollectionCache collectionCache = await this.cosmosQueryContext.QueryClient.GetCollectionCacheAsync(); @@ -145,11 +145,6 @@ private async Task GetContainerSettingsAsync(Cancellati { containerSettings = await collectionCache.ResolveCollectionAsync(request, cancellationToken); } - - } - else - { - containerSettings = null; } if (containerSettings == null) @@ -263,6 +258,13 @@ public static async Task CreateSpecializedDocumentQ string collectionRid, CancellationToken cancellationToken) { + if (!string.IsNullOrEmpty(partitionedQueryExecutionInfo.QueryInfo?.RewrittenQuery)) + { + cosmosQueryContext.SqlQuerySpec = new SqlQuerySpec( + partitionedQueryExecutionInfo.QueryInfo.RewrittenQuery, + cosmosQueryContext.SqlQuerySpec.Parameters); + } + // Figure out the optimal page size. long initialPageSize = cosmosQueryContext.QueryRequestOptions.MaxItemCount.GetValueOrDefault(ParallelQueryConfig.GetConfig().ClientInternalPageSize); @@ -346,6 +348,7 @@ internal static async Task> GetTargetPartitionKeyRanges( List targetRanges; if (queryRequestOptions.PartitionKey != null) { + // Dis-ambiguate the NonePK if used PartitionKeyInternal partitionKeyInternal = null; if (Object.ReferenceEquals(queryRequestOptions.PartitionKey, CosmosContainerSettings.NonePartitionKeyValue)) { diff --git a/Microsoft.Azure.Cosmos/src/Query/ExecutionComponent/TakeDocumentQueryExecutionComponent.cs b/Microsoft.Azure.Cosmos/src/Query/ExecutionComponent/TakeDocumentQueryExecutionComponent.cs index 80c12d612f..9e70686445 100644 --- a/Microsoft.Azure.Cosmos/src/Query/ExecutionComponent/TakeDocumentQueryExecutionComponent.cs +++ b/Microsoft.Azure.Cosmos/src/Query/ExecutionComponent/TakeDocumentQueryExecutionComponent.cs @@ -7,6 +7,7 @@ namespace Microsoft.Azure.Cosmos.Query.ExecutionComponent using System.Collections.Generic; using System.Globalization; using System.Linq; + using System.Net; using System.Threading; using System.Threading.Tasks; using Microsoft.Azure.Cosmos; @@ -49,7 +50,7 @@ public static async Task CreateLimitDocumen if (limitContinuationToken.Limit > limitCount) { - throw new BadRequestException($"limit count in continuation token: {limitContinuationToken.Limit} can not be greater than the limit count in the query: {limitCount}."); + throw new CosmosException(HttpStatusCode.BadRequest, $"limit count in continuation token: {limitContinuationToken.Limit} can not be greater than the limit count in the query: {limitCount}."); } return new TakeDocumentQueryExecutionComponent( @@ -75,7 +76,7 @@ public static async Task CreateTopDocumentQ if (topContinuationToken.Top > topCount) { - throw new BadRequestException($"top count in continuation token: {topContinuationToken.Top} can not be greater than the top count in the query: {topCount}."); + throw new CosmosException(HttpStatusCode.BadRequest, $"top count in continuation token: {topContinuationToken.Top} can not be greater than the top count in the query: {topCount}."); } return new TakeDocumentQueryExecutionComponent( diff --git a/Microsoft.Azure.Cosmos/src/Query/ParallelQuery/ItemProducer.cs b/Microsoft.Azure.Cosmos/src/Query/ParallelQuery/ItemProducer.cs index c6d637043c..2e3b7cdc2f 100644 --- a/Microsoft.Azure.Cosmos/src/Query/ParallelQuery/ItemProducer.cs +++ b/Microsoft.Azure.Cosmos/src/Query/ParallelQuery/ItemProducer.cs @@ -401,7 +401,7 @@ private void PopulatePartitionKeyRangeInfo(CosmosRequestMessage request) if (this.queryContext.ResourceTypeEnum.IsPartitioned()) { // If the request already has the logical partition key, - // then we shouldn't add the physical partiton key range id. + // then we shouldn't add the physical partition key range id. bool hasPartitionKey = request.Headers.Get(HttpConstants.HttpHeaders.PartitionKey) != null; if (!hasPartitionKey) diff --git a/Microsoft.Azure.Cosmos/src/Query/QueryPartitionProvider.cs b/Microsoft.Azure.Cosmos/src/Query/QueryPartitionProvider.cs index e0c8865659..4e4949a724 100644 --- a/Microsoft.Azure.Cosmos/src/Query/QueryPartitionProvider.cs +++ b/Microsoft.Azure.Cosmos/src/Query/QueryPartitionProvider.cs @@ -219,9 +219,10 @@ internal PartitionedQueryExecutionInfoInternal GetPartitionedQueryExecutionInfoI if (exception != null) { DefaultTrace.TraceInformation("QueryEngineConfiguration: " + this.queryengineConfiguration); - throw new BadRequestException( - "Message: " + serializedQueryExecutionInfo, - exception); + + throw new CosmosException( + HttpStatusCode.BadRequest, + "Message: " + serializedQueryExecutionInfo); } PartitionedQueryExecutionInfoInternal queryInfoInternal = diff --git a/Microsoft.Azure.Cosmos/src/Resource/CosmosException.cs b/Microsoft.Azure.Cosmos/src/Resource/CosmosException.cs index b884f575bd..4c50bc4023 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/CosmosException.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/CosmosException.cs @@ -17,6 +17,16 @@ public class CosmosException : Exception { private readonly CosmosResponseMessageHeaders Headers = null; + internal CosmosException( + HttpStatusCode statusCode, + string message, + Error error = null) : + base(message) + { + this.StatusCode = statusCode; + this.Error = error; + } + internal CosmosException( CosmosResponseMessage cosmosResponseMessage, string message, @@ -30,7 +40,6 @@ internal CosmosException( this.ActivityId = this.Headers?.GetHeaderValue(HttpConstants.HttpHeaders.ActivityId); this.RequestCharge = this.Headers == null ? 0 : this.Headers.GetHeaderValue(HttpConstants.HttpHeaders.RequestCharge); this.SubStatusCode = (int)this.Headers.SubStatusCode; - this.Error = error; if (cosmosResponseMessage.Headers.ContentLengthAsLong > 0) { using (StreamReader responseReader = new StreamReader(cosmosResponseMessage.Content)) @@ -39,6 +48,8 @@ internal CosmosException( } } } + + this.Error = error; } /// diff --git a/Microsoft.Azure.Cosmos/src/Resource/QueryResponses/FeedIteratorCore.cs b/Microsoft.Azure.Cosmos/src/Resource/QueryResponses/FeedIteratorCore.cs index 9f3c8fcc36..0fa0d5def7 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/QueryResponses/FeedIteratorCore.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/QueryResponses/FeedIteratorCore.cs @@ -63,16 +63,12 @@ internal FeedIteratorCore( /// /// (Optional) representing request cancellation. /// A query response from cosmos service - public override Task FetchNextSetAsync(CancellationToken cancellationToken = default(CancellationToken)) + public override async Task FetchNextSetAsync(CancellationToken cancellationToken = default(CancellationToken)) { - return this.nextResultSetDelegate(this.MaxItemCount, this.continuationToken, this.queryOptions, this.state, cancellationToken) - .ContinueWith(task => - { - CosmosResponseMessage response = task.Result; - this.continuationToken = response.Headers.Continuation; - this.HasMoreResults = GetHasMoreResults(this.continuationToken, response.StatusCode); - return response; - }, cancellationToken); + CosmosResponseMessage response = await this.nextResultSetDelegate(this.MaxItemCount, this.continuationToken, this.queryOptions, this.state, cancellationToken); + this.continuationToken = response.Headers.Continuation; + this.HasMoreResults = GetHasMoreResults(this.continuationToken, response.StatusCode); + return response; } internal static string GetContinuationToken(CosmosResponseMessage httpResponseMessage) @@ -149,19 +145,15 @@ internal FeedIteratorCore( /// /// (Optional) representing request cancellation. /// A query response from cosmos service - public override Task> FetchNextSetAsync(CancellationToken cancellationToken = default(CancellationToken)) + public override async Task> FetchNextSetAsync(CancellationToken cancellationToken = default(CancellationToken)) { cancellationToken.ThrowIfCancellationRequested(); - return this.nextResultSetDelegate(this.MaxItemCount, this.continuationToken, this.queryOptions, this.state, cancellationToken) - .ContinueWith(task => - { - FeedResponse response = task.Result; - this.HasMoreResults = response.HasMoreResults; - this.continuationToken = response.InternalContinuationToken; - - return response; - }, cancellationToken); + FeedResponse response = await this.nextResultSetDelegate(this.MaxItemCount, this.continuationToken, this.queryOptions, this.state, cancellationToken); + this.HasMoreResults = response.HasMoreResults; + this.continuationToken = response.InternalContinuationToken; + return response; + } internal static ReadFeedResponse CreateCosmosQueryResponse( diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosItemTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosItemTests.cs index 2d2f40b6b8..c6052bc7fc 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosItemTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosItemTests.cs @@ -731,21 +731,9 @@ public async Task NegativeQueryTest() await resultSet.FetchNextSetAsync(); Assert.Fail("Expected query to fail"); } - catch (Exception e) + catch (CosmosException exception) when (exception.StatusCode == HttpStatusCode.BadRequest) { - CosmosException exception = e.InnerException as CosmosException; - - if (exception == null) - { - throw e; - } - - if (exception.StatusCode != HttpStatusCode.BadRequest) - { - throw e; - } - - Assert.IsTrue(exception.Message.Contains("continuation token limit specified is not large enough")); + Assert.IsTrue(exception.Message.Contains("continuation token limit specified is not large enough"), exception.Message); } try @@ -757,21 +745,9 @@ public async Task NegativeQueryTest() await resultSet.FetchNextSetAsync(); Assert.Fail("Expected query to fail"); } - catch (AggregateException e) + catch (CosmosException exception) when (exception.StatusCode == HttpStatusCode.BadRequest) { - CosmosException exception = e.InnerException as CosmosException; - - if (exception == null) - { - throw e; - } - - if (exception.StatusCode != HttpStatusCode.BadRequest) - { - throw e; - } - - Assert.IsTrue(exception.Message.Contains("Syntax error, incorrect syntax near")); + Assert.IsTrue(exception.Message.Contains("Syntax error, incorrect syntax near"), exception.Message); } } @@ -970,6 +946,14 @@ private async Task> CreateRandomItems(int pkCount, int perPK } private async Task CreateNonPartitionedContainer() + { + await CosmosItemTests.CreateNonPartitionedContainer(this.database.Id, + CosmosItemTests.nonPartitionContainerId); + } + + internal static async Task CreateNonPartitionedContainer( + string dbName, + string containerName) { string authKey = ConfigurationManager.AppSettings["MasterKey"]; string endpoint = ConfigurationManager.AppSettings["GatewayEndpoint"]; @@ -978,18 +962,19 @@ private async Task CreateNonPartitionedContainer() Uri baseUri = new Uri(endpoint); string verb = "POST"; string resourceType = "colls"; - string resourceId = string.Format("dbs/{0}", this.database.Id); - string resourceLink = string.Format("dbs/{0}/colls", this.database.Id); + string resourceId = string.Format("dbs/{0}", dbName); + string resourceLink = string.Format("dbs/{0}/colls", dbName); client.DefaultRequestHeaders.Add("x-ms-date", utc_date); client.DefaultRequestHeaders.Add("x-ms-version", CosmosItemTests.PreNonPartitionedMigrationApiVersion); - string authHeader = this.GenerateMasterKeyAuthorizationSignature(verb, resourceId, resourceType, authKey, "master", "1.0"); + string authHeader = CosmosItemTests.GenerateMasterKeyAuthorizationSignature(verb, resourceId, resourceType, authKey, "master", "1.0"); client.DefaultRequestHeaders.Add("authorization", authHeader); - string containerDefinition = "{\n \"id\": \"" + nonPartitionContainerId + "\"\n}"; + string containerDefinition = "{\n \"id\": \"" + containerName + "\"\n}"; StringContent containerContent = new StringContent(containerDefinition); Uri requestUri = new Uri(baseUri, resourceLink); - await client.PostAsync(requestUri.ToString(), containerContent); + HttpResponseMessage response = await client.PostAsync(requestUri.ToString(), containerContent); + Assert.AreEqual(HttpStatusCode.Created, response.StatusCode, response.ToString()); } private async Task CreateItemInNonPartitionedContainer(string itemId) @@ -1003,16 +988,19 @@ private async Task CreateItemInNonPartitionedContainer(string itemId) string resourceType = "docs"; string resourceId = string.Format("dbs/{0}/colls/{1}", this.database.Id, nonPartitionContainerId); string resourceLink = string.Format("dbs/{0}/colls/{1}/docs", this.database.Id, nonPartitionContainerId); - string authHeader = this.GenerateMasterKeyAuthorizationSignature(verb, resourceId, resourceType, authKey, "master", "1.0"); + string authHeader = CosmosItemTests.GenerateMasterKeyAuthorizationSignature(verb, resourceId, resourceType, authKey, "master", "1.0"); client.DefaultRequestHeaders.Add("x-ms-date", utc_date); client.DefaultRequestHeaders.Add("x-ms-version", CosmosItemTests.PreNonPartitionedMigrationApiVersion); client.DefaultRequestHeaders.Add("authorization", authHeader); - string itemDefinition = JsonConvert.SerializeObject(this.CreateRandomToDoActivity(id: itemId)); - StringContent itemContent = new StringContent(itemDefinition); - Uri requestUri = new Uri(baseUri, resourceLink); - await client.PostAsync(requestUri.ToString(), itemContent); + string itemDefinition = JsonConvert.SerializeObject(this.CreateRandomToDoActivity(id: nonPartitionItemId)); + { + StringContent itemContent = new StringContent(itemDefinition); + Uri requestUri = new Uri(baseUri, resourceLink); + HttpResponseMessage response = await client.PostAsync(requestUri.ToString(), itemContent); + Assert.AreEqual(HttpStatusCode.Created, response.StatusCode, response.ToString()); + } } private async Task CreateUndefinedPartitionItem() @@ -1035,7 +1023,7 @@ private async Task CreateUndefinedPartitionItem() resourceType = "docs"; resourceId = string.Format("dbs/{0}/colls/{1}", this.database.Id, this.Container.Id); resourceLink = string.Format("dbs/{0}/colls/{1}/docs", this.database.Id, this.Container.Id); - string authHeader = this.GenerateMasterKeyAuthorizationSignature(verb, resourceId, resourceType, authKey, "master", "1.0"); + string authHeader = CosmosItemTests.GenerateMasterKeyAuthorizationSignature(verb, resourceId, resourceType, authKey, "master", "1.0"); client.DefaultRequestHeaders.Remove("authorization"); client.DefaultRequestHeaders.Add("authorization", authHeader); @@ -1047,7 +1035,7 @@ private async Task CreateUndefinedPartitionItem() await client.PostAsync(requestUri.ToString(), itemContent); } - private string GenerateMasterKeyAuthorizationSignature(string verb, string resourceId, string resourceType, string key, string keyType, string tokenVersion) + private static string GenerateMasterKeyAuthorizationSignature(string verb, string resourceId, string resourceType, string key, string keyType, string tokenVersion) { System.Security.Cryptography.HMACSHA256 hmacSha256 = new System.Security.Cryptography.HMACSHA256 { Key = Convert.FromBase64String(key) }; diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CrossPartitionQueryTests.OnePartition.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CrossPartitionQueryTests.OnePartition.cs new file mode 100644 index 0000000000..bbbdfa6305 --- /dev/null +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CrossPartitionQueryTests.OnePartition.cs @@ -0,0 +1,3949 @@ +//----------------------------------------------------------------------- +// +// Copyright (c) Microsoft Corporation. All rights reserved. +// +//----------------------------------------------------------------------- +namespace Microsoft.Azure.Cosmos.SDK.EmulatorTests +{ + using System; + using System.Collections.Generic; + using System.Collections.ObjectModel; + using System.Diagnostics; + using System.Globalization; + using System.IO; + using System.Linq; + using System.Net; + using System.Text; + using System.Text.RegularExpressions; + using System.Threading; + using System.Threading.Tasks; + using System.Xml; + using Microsoft.Azure.Cosmos.CosmosElements; + using Microsoft.Azure.Cosmos.Routing; + using Microsoft.Azure.Documents; + using Microsoft.Azure.Documents.Routing; + using Microsoft.VisualStudio.TestTools.UnitTesting; + using Newtonsoft.Json; + using Newtonsoft.Json.Converters; + using Newtonsoft.Json.Linq; + using Query; + using Query.ParallelQuery; + + /// + /// Tests for CrossPartitionQueryTestsOnePartition. + /// + [TestCategory("Quarantine")] + [TestClass] + public class CrossPartitionQueryTestsOnePartition + { + private static readonly string[] NoDocuments = new string[] { }; + private CosmosClient GatewayClient = TestCommon.CreateCosmosClient(true); + private CosmosClient Client = TestCommon.CreateCosmosClient(false); + private CosmosDatabase database; + private readonly AsyncLocal responseLengthBytes = new AsyncLocal(); + private readonly AsyncLocal outerFeedResponseActivityId = new AsyncLocal(); + + [FlagsAttribute] + private enum ConnectionModes + { + None = 0, + Direct = 0x1, + Gateway = 0x2, + } + + [TestInitialize] + public async Task Initialize() + { + await this.CleanUp(); + this.database = await this.Client.Databases.CreateDatabaseAsync(Guid.NewGuid().ToString() + "db"); + } + + [TestCleanup] + public async Task Cleanup() + { + await this.database.DeleteAsync(); + } + + private static string GetApiVersion() + { + return HttpConstants.Versions.CurrentVersion; + } + + private static void SetApiVersion(string apiVersion) + { + HttpConstants.Versions.CurrentVersion = apiVersion; + HttpConstants.Versions.CurrentVersionUTF8 = Encoding.UTF8.GetBytes(apiVersion); + } + + private async Task> GetPartitionKeyRanges(CosmosContainerSettings container) + { + Range fullRange = new Range( + PartitionKeyInternal.MinimumInclusiveEffectivePartitionKey, + PartitionKeyInternal.MaximumExclusiveEffectivePartitionKey, + true, + false); + IRoutingMapProvider routingMapProvider = await this.Client.DocumentClient.GetPartitionKeyRangeCacheAsync(); + IReadOnlyList ranges = await routingMapProvider.TryGetOverlappingRangesAsync(container.ResourceId, fullRange); + return ranges; + } + + private async Task CreatePartitionContainer(string partitionKey = "/id", Microsoft.Azure.Cosmos.IndexingPolicy indexingPolicy = null) + { + ContainerResponse containerResponse = await this.database.Containers.CreateContainerAsync( + new CosmosContainerSettings + { + Id = Guid.NewGuid().ToString() + "container", + IndexingPolicy = indexingPolicy == null ? new Cosmos.IndexingPolicy + { + IncludedPaths = new Collection + { + new Cosmos.IncludedPath + { + Path = "/*", + Indexes = new Collection + { + Cosmos.Index.Range(Cosmos.DataType.Number), + Cosmos.Index.Range(Cosmos.DataType.String), + } + } + } + } : indexingPolicy, + PartitionKey = new PartitionKeyDefinition + { + Paths = new Collection { partitionKey }, + Kind = PartitionKind.Hash + } + }, + // This throughput needs to be about half the max with multi master + // otherwise it will create about twice as many partitions. + 5000); + + IReadOnlyList ranges = await this.GetPartitionKeyRanges(containerResponse); + Assert.AreEqual(1, ranges.Count()); + + return containerResponse; + } + + private async Task>> CreatePartitionedContainerAndIngestDocuments(IEnumerable documents, string partitionKey = "/id", Cosmos.IndexingPolicy indexingPolicy = null) + { + CosmosContainer partitionedCollection = await this.CreatePartitionContainer(partitionKey, indexingPolicy); + List insertedDocuments = new List(); + string jObjectPartitionKey = partitionKey.Remove(0, 1); + foreach (string document in documents) + { + JObject documentObject = JsonConvert.DeserializeObject(document); + if (documentObject["id"] == null) + { + documentObject["id"] = Guid.NewGuid().ToString(); + } + + JValue pkToken = (JValue)documentObject[jObjectPartitionKey]; + object pkValue = pkToken != null ? pkToken.Value : Undefined.Value; + insertedDocuments.Add((await partitionedCollection.Items.CreateItemAsync(pkValue, documentObject)).Resource.ToObject()); + + } + + return new Tuple>(partitionedCollection, insertedDocuments); + } + + private async Task CleanUp() + { + FeedIterator allDatabases = this.Client.Databases.GetDatabaseIterator(); + + while (allDatabases.HasMoreResults) + { + foreach (CosmosDatabaseSettings db in await allDatabases.FetchNextSetAsync()) + { + await this.Client.Databases[db.Id].DeleteAsync(); + } + } + } + + private async Task RunWithApiVersion(string apiVersion, Func function) + { + string originalApiVersion = GetApiVersion(); + CosmosClient originalCosmosClient = this.Client; + CosmosClient originalGatewayClient = this.GatewayClient; + CosmosDatabase originalDatabase = this.database; + + try + { + SetApiVersion(apiVersion); + if (apiVersion != originalApiVersion) + { + this.Client = TestCommon.CreateCosmosClient(false); + this.GatewayClient = TestCommon.CreateCosmosClient(true); + this.database = this.Client.Databases[this.database.Id]; + } + + await function(); + } + finally + { + this.Client = originalCosmosClient; + this.GatewayClient = originalGatewayClient; + this.database = originalDatabase; + SetApiVersion(originalApiVersion); + } + } + + internal delegate Task Query( + CosmosContainer container, + IEnumerable documents); + + internal delegate Task Query( + CosmosContainer container, + IEnumerable documents, + T testArgs); + + internal delegate CosmosClient CosmosClientFactory(ConnectionMode connectionMode); + + private async Task CreateIngestQueryDelete( + ConnectionModes connectionModes, + IEnumerable documents, + Query query, + string partitionKey = "/id", + Cosmos.IndexingPolicy indexingPolicy = null, + CosmosClientFactory cosmosClientFactory = null) + { + Query queryWrapper = (container, inputDocuments, throwaway) => + { + return query(container, inputDocuments); + }; + + await this.CreateIngestQueryDelete( + connectionModes, + documents, + queryWrapper, + null, + partitionKey, + indexingPolicy, + cosmosClientFactory); + } + + private async Task CreateIngestQueryDelete( + ConnectionModes connectionModes, + IEnumerable documents, + Query query, + T testArgs, + string partitionKey = "/id", + Cosmos.IndexingPolicy indexingPolicy = null, + CosmosClientFactory cosmosClientFactory = null) + { + await this.CreateIngestQueryDelete( + connectionModes, + documents, + query, + cosmosClientFactory ?? this.CreateDefaultCosmosClient, + testArgs, + partitionKey, + indexingPolicy); + } + + /// + /// Task that wraps boiler plate code for query tests (container create -> ingest documents -> query documents -> delete collections). + /// Note that this function will take the cross product connectionModes + /// + /// The connection modes to use. + /// The documents to ingest + /// + /// The callback for the queries. + /// All the standard arguments will be passed in. + /// Please make sure that this function is idempotent, since a container will be reused for each connection mode. + /// + /// + /// The callback for the create CosmosClient. This is invoked for the different ConnectionModes that the query is targeting. + /// If CosmosClient instantiated by this does not apply the expected ConnectionMode, an assert is thrown. + /// + /// The partition key for the partition container. + /// The optional args that you want passed in to the query. + /// A task to await on. + private async Task CreateIngestQueryDelete( + ConnectionModes connectionModes, + IEnumerable documents, + Query query, + CosmosClientFactory cosmosClientFactory, + T testArgs, + string partitionKey = "/id", + Cosmos.IndexingPolicy indexingPolicy = null) + { + int retryCount = 1; + AggregateException exceptionHistory = new AggregateException(); + while (retryCount-- > 0) + { + try + { + List>>> createContainerTasks = new List>>> + { + this.CreatePartitionedContainerAndIngestDocuments(documents, partitionKey, indexingPolicy) + }; + + Tuple>[] collectionsAndDocuments = await Task.WhenAll(createContainerTasks); + + List cosmosClients = new List(); + foreach (ConnectionModes connectionMode in Enum.GetValues(connectionModes.GetType()).Cast().Where(connectionModes.HasFlag)) + { + if (connectionMode == ConnectionModes.None) + { + continue; + } + + ConnectionMode targetConnectionMode = GetTargetConnectionMode(connectionMode); + CosmosClient cosmosClient = cosmosClientFactory(targetConnectionMode); + + Assert.AreEqual(targetConnectionMode, cosmosClient.Configuration.ConnectionMode, "Test setup: Invalid connection policy applied to CosmosClient"); + cosmosClients.Add(cosmosClient); + } + + bool succeeded = false; + while (!succeeded) + { + try + { + List queryTasks = new List(); + foreach (CosmosClient cosmosClient in cosmosClients) + { + foreach (Tuple> containerAndDocuments in collectionsAndDocuments) + { + CosmosContainer container = cosmosClient.Databases[containerAndDocuments.Item1.Database.Id].Containers[containerAndDocuments.Item1.Id]; + queryTasks.Add(query(container, containerAndDocuments.Item2, testArgs)); + } + } + + await Task.WhenAll(queryTasks); + succeeded = true; + } + catch (TaskCanceledException) + { + // SDK throws TaskCanceledException every now and then + } + } + + List> deleteContainerTasks = new List>(); + foreach (CosmosContainer container in collectionsAndDocuments.Select(tuple => tuple.Item1)) + { + deleteContainerTasks.Add(container.DeleteAsync()); + } + + await Task.WhenAll(deleteContainerTasks); + + // If you made it here then it's all good + break; + } + catch (Exception ex) + { + if (ex.GetType() == typeof(AssertFailedException)) + { + throw; + } + else + { + List previousExceptions = exceptionHistory.InnerExceptions.ToList(); + previousExceptions.Add(ex); + exceptionHistory = new AggregateException(previousExceptions); + } + } + } + + if (exceptionHistory.InnerExceptions.Count > 0) + { + throw exceptionHistory; + } + } + + private static ConnectionMode GetTargetConnectionMode(ConnectionModes connectionMode) + { + ConnectionMode targetConnectionMode = ConnectionMode.Gateway; + switch (connectionMode) + { + case ConnectionModes.Gateway: + targetConnectionMode = ConnectionMode.Gateway; + break; + + case ConnectionModes.Direct: + targetConnectionMode = ConnectionMode.Direct; + break; + + default: + throw new ArgumentException($"Unexpected connection mode: {connectionMode}"); + } + + return targetConnectionMode; + } + + private CosmosClient CreateDefaultCosmosClient(ConnectionMode connectionMode) + { + switch (connectionMode) + { + case ConnectionMode.Gateway: + return this.GatewayClient; + case ConnectionMode.Direct: + return this.Client; + default: + throw new ArgumentException($"Unexpected connection mode: {connectionMode}"); + } + } + + private CosmosClient CreateNewCosmosClient(ConnectionMode connectionMode) + { + switch (connectionMode) + { + case ConnectionMode.Gateway: + return TestCommon.CreateCosmosClient(true); + case ConnectionMode.Direct: + return TestCommon.CreateCosmosClient(false); + default: + throw new ArgumentException($"Unexpected connection mode: {connectionMode}"); + } + } + + private static async Task> QueryWithContinuationTokens( + CosmosContainer container, + string query, + int maxItemCount, + QueryRequestOptions queryRequestOptions = null) + { + List results = new List(); + string continuationToken = null; + do + { + FeedIterator itemQuery = container.Items.CreateItemQuery( + sqlQueryText: query, + maxConcurrency: 2, + maxItemCount: maxItemCount, + requestOptions: queryRequestOptions, + continuationToken: continuationToken); + + FeedResponse FeedResponse = await itemQuery.FetchNextSetAsync(); + results.AddRange(FeedResponse); + continuationToken = FeedResponse.Continuation; + } while (continuationToken != null); + + return results; + } + + private static async Task> QueryWithoutContinuationTokens( + CosmosContainer container, + string query, + int maxItemCount, + QueryRequestOptions queryRequestOptions = null) + { + List results = new List(); + FeedIterator itemQuery = container.Items.CreateItemQuery( + sqlQueryText: query, + maxConcurrency: 2, + requestOptions: queryRequestOptions); + + while (itemQuery.HasMoreResults) + { + results.AddRange(await itemQuery.FetchNextSetAsync()); + } + + return results; + } + + private static async Task NoOp() + { + await Task.Delay(0); + } + + private async Task RandomlyThrowException(Exception exception = null) + { + await CrossPartitionQueryTestsOnePartition.NoOp(); + Random random = new Random(); + if (random.Next(0, 2) == 0) + { + throw exception; + } + } + + [TestMethod] + public async Task TestBadQueriesOverMultiplePartitions() + { + await this.CreateIngestQueryDelete( + ConnectionModes.Direct | ConnectionModes.Gateway, + CrossPartitionQueryTestsOnePartition.NoDocuments, + this.TestBadQueriesOverMultiplePartitionsHelper); + } + + [TestMethod] + public void TestContinuationTokenSerialization() + { + CompositeContinuationToken compositeContinuationToken = new CompositeContinuationToken() + { + Token = "asdf", + Range = new Range("asdf", "asdf", false, false), + }; + + string serializedCompositeContinuationToken = JsonConvert.SerializeObject(compositeContinuationToken); + CompositeContinuationToken deserializedCompositeContinuationToken = JsonConvert.DeserializeObject(serializedCompositeContinuationToken); + Assert.AreEqual(compositeContinuationToken.Token, deserializedCompositeContinuationToken.Token); + //Assert.IsTrue(compositeContinuationToken.Range.Equals(deserializedCompositeContinuationToken.Range)); + + + string orderByItemSerialized = @"{""item"" : 1337 }"; + byte[] bytes = Encoding.UTF8.GetBytes(orderByItemSerialized); + OrderByItem orderByItem = new OrderByItem(CosmosElement.Create(bytes)); + OrderByContinuationToken orderByContinuationToken = new OrderByContinuationToken( + compositeContinuationToken, + new List { orderByItem }, + "asdf", + 42, + "asdf"); + string serializedOrderByContinuationToken = JsonConvert.SerializeObject(orderByContinuationToken); + OrderByContinuationToken deserializedOrderByContinuationToken = JsonConvert.DeserializeObject(serializedOrderByContinuationToken); + Assert.AreEqual( + orderByContinuationToken.CompositeContinuationToken.Token, + deserializedOrderByContinuationToken.CompositeContinuationToken.Token); + //Assert.IsTrue( + // orderByContinuationToken.CompositeContinuationToken.Range.Equals( + // deserializedOrderByContinuationToken.CompositeContinuationToken.Range)); + Assert.IsTrue(CosmosElementEqualityComparer.Value.Equals(orderByContinuationToken.OrderByItems[0].Item, deserializedOrderByContinuationToken.OrderByItems[0].Item)); + Assert.AreEqual(orderByContinuationToken.Rid, deserializedOrderByContinuationToken.Rid); + Assert.AreEqual(orderByContinuationToken.SkipCount, deserializedOrderByContinuationToken.SkipCount); + } + + private async Task TestBadQueriesOverMultiplePartitionsHelper(CosmosContainer container, IEnumerable documents) + { + await CrossPartitionQueryTestsOnePartition.NoOp(); + try + { + FeedIterator resultSetIterator = container.Items.CreateItemQuery( + @"SELECT * FROM Root r WHERE a = 1", + maxConcurrency: 2); + + await resultSetIterator.FetchNextSetAsync(); + + Assert.Fail("Expected CosmosException"); + } + catch (CosmosException e) when (e.StatusCode == HttpStatusCode.BadRequest) + { + if (!e.Message.StartsWith("Message: {\"errors\":[{\"severity\":\"Error\",\"location\":{\"start\":27,\"end\":28},\"code\":\"SC2001\",\"message\":\"Identifier 'a' could not be resolved.\"}]}")) + { + throw e; + } + } + } + + /// + //"SELECT c._ts, c.id, c.TicketNumber, c.PosCustomerNumber, c.CustomerId, c.CustomerUserId, c.ContactEmail, c.ContactPhone, c.StoreCode, c.StoreUid, c.PoNumber, c.OrderPlacedOn, c.OrderType, c.OrderStatus, c.Customer.UserFirstName, c.Customer.UserLastName, c.Customer.Name, c.UpdatedBy, c.UpdatedOn, c.ExpirationDate, c.TotalAmountFROM c ORDER BY c._ts"' created an ArgumentOutofRangeException since ServiceInterop was returning DISP_E_BUFFERTOOSMALL in the case of an invalid query that is also really long. + /// This test case just double checks that you get the appropriate document client exception instead of just failing. + /// + [TestMethod] + public async Task TestQueryCrossParitionPartitionProviderInvalid() + { + await this.CreateIngestQueryDelete( + ConnectionModes.Direct | ConnectionModes.Gateway, + CrossPartitionQueryTestsOnePartition.NoDocuments, + this.TestQueryCrossParitionPartitionProviderInvalidHelper); + } + + private async Task TestQueryCrossParitionPartitionProviderInvalidHelper(CosmosContainer container, IEnumerable documents) + { + await CrossPartitionQueryTestsOnePartition.NoOp(); + try + { + /// note that there is no space before the from clause thus this query should fail + /// '"code":"SC2001","message":"Identifier 'c' could not be resolved."' + string query = "SELECT c._ts, c.id, c.TicketNumber, c.PosCustomerNumber, c.CustomerId, c.CustomerUserId, c.ContactEmail, c.ContactPhone, c.StoreCode, c.StoreUid, c.PoNumber, c.OrderPlacedOn, c.OrderType, c.OrderStatus, c.Customer.UserFirstName, c.Customer.UserLastName, c.Customer.Name, c.UpdatedBy, c.UpdatedOn, c.ExpirationDate, c.TotalAmountFROM c ORDER BY c._ts"; + List expectedValues = new List(); + FeedIterator resultSetIterator = container.Items.CreateItemQuery( + query, + maxConcurrency: 0); + + while (resultSetIterator.HasMoreResults) + { + expectedValues.AddRange(await resultSetIterator.FetchNextSetAsync()); + } + + Assert.Fail("Expected to get an exception for this query."); + } + catch (CosmosException e) when (e.StatusCode == HttpStatusCode.BadRequest) + { + } + } + + [TestMethod] + public async Task TestQueryAndReadFeedWithPartitionKey() + { + string[] documents = new[] + { + @"{""id"":""documentId1"",""key"":""A"",""prop"":3,""shortArray"":[{""a"":5}]}", + @"{""id"":""documentId2"",""key"":""A"",""prop"":2,""shortArray"":[{""a"":6}]}", + @"{""id"":""documentId3"",""key"":""A"",""prop"":1,""shortArray"":[{""a"":7}]}", + @"{""id"":""documentId4"",""key"":5,""prop"":3,""shortArray"":[{""a"":5}]}", + @"{""id"":""documentId5"",""key"":5,""prop"":2,""shortArray"":[{""a"":6}]}", + @"{""id"":""documentId6"",""key"":5,""prop"":1,""shortArray"":[{""a"":7}]}", + @"{""id"":""documentId10"",""prop"":3,""shortArray"":[{""a"":5}]}", + @"{""id"":""documentId11"",""prop"":2,""shortArray"":[{""a"":6}]}", + @"{""id"":""documentId12"",""prop"":1,""shortArray"":[{""a"":7}]}", + }; + + await this.CreateIngestQueryDelete( + ConnectionModes.Direct | ConnectionModes.Gateway, + documents, + this.TestQueryAndReadFeedWithPartitionKeyHelper, + "/key"); + } + + private async Task TestQueryAndReadFeedWithPartitionKeyHelper( + CosmosContainer container, + IEnumerable documents) + { + Assert.AreEqual(0, (await this.RunQuery( + container, + @"SELECT * FROM Root r WHERE false", + maxConcurrency: 1)).Count); + + object[] keys = new object[] { "A", 5, Undefined.Value }; + for (int i = 0; i < keys.Length; ++i) + { + List expected = documents.Skip(i * 3).Take(3).Select(doc => doc.Id).ToList(); + string expectedResult = string.Join(",", expected); + // Order-by + expected.Reverse(); + string expectedOrderByResult = string.Join(",", expected); + + List<(string, string)> queries = new List<(string, string)>() + { + ($@"SELECT * FROM Root r WHERE r.id IN (""{expected[0]}"", ""{expected[1]}"", ""{expected[2]}"")", expectedResult), + (@"SELECT * FROM Root r WHERE r.prop BETWEEN 1 AND 3", expectedResult), + (@"SELECT VALUE r FROM Root r JOIN c IN r.shortArray WHERE c.a BETWEEN 5 and 7", expectedResult), + ($@"SELECT TOP 10 * FROM Root r WHERE r.id IN (""{expected[0]}"", ""{expected[1]}"", ""{expected[2]}"")", expectedResult), + (@"SELECT TOP 10 * FROM Root r WHERE r.prop BETWEEN 1 AND 3", expectedResult), + (@"SELECT TOP 10 VALUE r FROM Root r JOIN c IN r.shortArray WHERE c.a BETWEEN 5 and 7", expectedResult), + ($@"SELECT * FROM Root r WHERE r.id IN (""{expected[0]}"", ""{expected[1]}"", ""{expected[2]}"") ORDER BY r.prop", expectedOrderByResult), + (@"SELECT * FROM Root r WHERE r.prop BETWEEN 1 AND 3 ORDER BY r.prop", expectedOrderByResult), + (@"SELECT VALUE r FROM Root r JOIN c IN r.shortArray WHERE c.a BETWEEN 5 and 7 ORDER BY r.prop", expectedOrderByResult), + }; + + + + if (i < keys.Length - 1) + { + string key; + if (keys[i] is string) + { + key = "'" + keys[i].ToString() + "'"; + } + else + { + key = keys[i].ToString(); + } + + queries.Add((string.Format(CultureInfo.InvariantCulture, @"SELECT * FROM Root r WHERE r.key = {0} ORDER BY r.prop", key), expectedOrderByResult)); + } + + foreach ((string, string) queryAndExpectedResult in queries) + { + FeedIterator resultSetIterator = container.Items.CreateItemQuery( + sqlQueryText: queryAndExpectedResult.Item1, + partitionKey: keys[i], + maxItemCount: 1); + + List result = new List(); + while (resultSetIterator.HasMoreResults) + { + result.AddRange(await resultSetIterator.FetchNextSetAsync()); + } + + string resultDocIds = string.Join(",", result.Select(doc => doc.Id)); + Assert.AreEqual(queryAndExpectedResult.Item2, resultDocIds); + } + } + } + + [TestMethod] + public async Task TestQueryMultiplePartitionsSinglePartitionKey() + { + string[] documents = new[] + { + @"{""pk"":""doc1""}", + @"{""pk"":""doc2""}", + @"{""pk"":""doc3""}", + @"{""pk"":""doc4""}", + @"{""pk"":""doc5""}", + @"{""pk"":""doc6""}", + }; + + await this.CreateIngestQueryDelete( + ConnectionModes.Direct | ConnectionModes.Gateway, + documents, + this.TestQueryMultiplePartitionsSinglePartitionKeyHelper, + "/pk"); + } + + private async Task TestQueryMultiplePartitionsSinglePartitionKeyHelper(CosmosContainer container, IEnumerable documents) + { + // Query with partition key should be done in one round trip. + FeedIterator resultSetIterator = container.Items.CreateItemQuery( + "SELECT * FROM c WHERE c.pk = 'doc5'", + partitionKey: "doc5"); + + FeedResponse response = await resultSetIterator.FetchNextSetAsync(); + Assert.AreEqual(1, response.Count()); + Assert.IsNull(response.Continuation); + + resultSetIterator = container.Items.CreateItemQuery( + "SELECT * FROM c WHERE c.pk = 'doc10'", + partitionKey: "doc10"); + + response = await resultSetIterator.FetchNextSetAsync(); + Assert.AreEqual(0, response.Count()); + Assert.IsNull(response.Continuation); + } + + private struct QueryWithSpecialPartitionKeysArgs + { + public string Name; + public object Value; + public Func ValueToPartitionKey; + } + + // V3 only supports Numeric, string, bool, null, undefined + [TestMethod] + [Ignore] + public async Task TestQueryWithSpecialPartitionKeys() + { + await CrossPartitionQueryTestsOnePartition.NoOp(); + QueryWithSpecialPartitionKeysArgs[] queryWithSpecialPartitionKeyArgsList = new QueryWithSpecialPartitionKeysArgs[] + { + new QueryWithSpecialPartitionKeysArgs() + { + Name = "Guid", + Value = Guid.NewGuid(), + ValueToPartitionKey = val => val.ToString(), + }, + //new QueryWithSpecialPartitionKeysArgs() + //{ + // Name = "DateTime", + // Value = DateTime.Now, + // ValueToPartitionKey = val => + // { + // string str = JsonConvert.SerializeObject( + // val, + // new JsonSerializerSettings() + // { + // Converters = new List { new IsoDateTimeConverter() } + // }); + // return str.Substring(1, str.Length - 2); + // }, + //}, + new QueryWithSpecialPartitionKeysArgs() + { + Name = "Enum", + Value = HttpStatusCode.OK, + ValueToPartitionKey = val => (int)val, + }, + new QueryWithSpecialPartitionKeysArgs() + { + Name = "CustomEnum", + Value = HttpStatusCode.OK, + ValueToPartitionKey = val => val.ToString(), + }, + new QueryWithSpecialPartitionKeysArgs() + { + Name = "ResourceId", + Value = "testid", + ValueToPartitionKey = val => val, + }, + new QueryWithSpecialPartitionKeysArgs() + { + Name = "CustomDateTime", + Value = new DateTime(2016, 11, 12), + ValueToPartitionKey = val => EpochDateTimeConverter.DateTimeToEpoch((DateTime)val), + }, + }; + + foreach (QueryWithSpecialPartitionKeysArgs testArg in queryWithSpecialPartitionKeyArgsList) + { + // For this test we need to split direct and gateway runs into separate collections, + // since the query callback inserts some documents (thus has side effects). + await this.CreateIngestQueryDelete( + ConnectionModes.Direct, + CrossPartitionQueryTestsOnePartition.NoDocuments, + this.TestQueryWithSpecialPartitionKeysHelper, + testArg, + "/" + testArg.Name); + + await this.CreateIngestQueryDelete( + ConnectionModes.Gateway, + CrossPartitionQueryTestsOnePartition.NoDocuments, + this.TestQueryWithSpecialPartitionKeysHelper, + testArg, + "/" + testArg.Name); + } + } + + private async Task TestQueryWithSpecialPartitionKeysHelper(CosmosContainer container, IEnumerable documents, QueryWithSpecialPartitionKeysArgs testArgs) + { + QueryWithSpecialPartitionKeysArgs args = testArgs; + + SpecialPropertyDocument specialPropertyDocument = new SpecialPropertyDocument + { + id = Guid.NewGuid().ToString() + }; + + specialPropertyDocument.GetType().GetProperty(args.Name).SetValue(specialPropertyDocument, args.Value); + Func getPropertyValueFunction = d => d.GetType().GetProperty(args.Name).GetValue(d); + + ItemResponse response = await container.Items.CreateItemAsync(testArgs.Value, specialPropertyDocument); + dynamic returnedDoc = response.Resource; + Assert.AreEqual(args.Value, getPropertyValueFunction((SpecialPropertyDocument)returnedDoc)); + + PartitionKey key = new PartitionKey(args.ValueToPartitionKey(args.Value)); + response = await container.Items.ReadItemAsync(key, response.Resource.id); + returnedDoc = response.Resource; + Assert.AreEqual(args.Value, getPropertyValueFunction((SpecialPropertyDocument)returnedDoc)); + + returnedDoc = (await this.RunSinglePartitionQuery( + container, + "SELECT * FROM t", + key)).Single(); + + Assert.AreEqual(args.Value, getPropertyValueFunction(returnedDoc)); + + string query; + switch (args.Name) + { + case "Guid": + query = $"SELECT * FROM T WHERE T.Guid = '{(Guid)args.Value}'"; + break; + case "Enum": + query = $"SELECT * FROM T WHERE T.Enum = '{(HttpStatusCode)args.Value}'"; + break; + case "DateTime": + query = $"SELECT * FROM T WHERE T.DateTime = '{(DateTime)args.Value}'"; + break; + case "CustomEnum": + query = $"SELECT * FROM T WHERE T.CustomEnum = '{(HttpStatusCode)args.Value}'"; + break; + case "ResourceId": + query = $"SELECT * FROM T WHERE T.ResourceId = '{(string)args.Value}'"; + break; + case "CustomDateTime": + query = $"SELECT * FROM T WHERE T.CustomDateTime = '{(DateTime)args.Value}'"; + break; + default: + query = null; + break; + } + + returnedDoc = (await container.Items.CreateItemQuery( + query, + partitionKey: args.ValueToPartitionKey, + maxItemCount: 1).FetchNextSetAsync()).First(); + + Assert.AreEqual(args.Value, getPropertyValueFunction(returnedDoc)); + } + + private sealed class SpecialPropertyDocument + { + public string id + { + get; + set; + } + + public Guid Guid + { + get; + set; + } + + [JsonConverter(typeof(IsoDateTimeConverter))] + public DateTime DateTime + { + get; + set; + } + + [JsonConverter(typeof(EpochDateTimeConverter))] + public DateTime CustomDateTime + { + get; + set; + } + + + public HttpStatusCode Enum + { + get; + set; + } + + [JsonConverter(typeof(StringEnumConverter))] + public HttpStatusCode CustomEnum + { + get; + set; + } + + public string ResourceId + { + get; + set; + } + } + + private sealed class EpochDateTimeConverter : JsonConverter + { + public static int DateTimeToEpoch(DateTime dt) + { + if (!dt.Equals(DateTime.MinValue)) + { + DateTime epoch = new DateTime(1970, 1, 1); + TimeSpan epochTimeSpan = dt - epoch; + return (int)epochTimeSpan.TotalSeconds; + } + else + { + return int.MinValue; + } + } + + public override bool CanConvert(Type objectType) + { + return true; + } + + public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer) + { + if (reader.TokenType == JsonToken.None || reader.TokenType == JsonToken.Null) + { + return null; + } + + + if (reader.TokenType != JsonToken.Integer) + { + throw new Exception( + string.Format( + CultureInfo.InvariantCulture, + "Unexpected token parsing date. Expected Integer, got {0}.", + reader.TokenType)); + } + + int seconds = Convert.ToInt32(reader.Value, CultureInfo.InvariantCulture); + return new DateTime(1970, 1, 1).AddSeconds(seconds); + } + + public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer) + { + int seconds; + if (value is DateTime) + { + seconds = DateTimeToEpoch((DateTime)value); + } + else + { + throw new Exception("Expected date object value."); + } + + writer.WriteValue(seconds); + } + } + + private struct QueryCrossPartitionWithLargeNumberOfKeysArgs + { + public int NumberOfDocuments; + public string PartitionKey; + public HashSet ExpectedPartitionKeyValues; + } + + [TestMethod] + public async Task TestQueryCrossPartitionWithLargeNumberOfKeys() + { + int numberOfDocuments = 1000; + string partitionKey = "key"; + HashSet expectedPartitionKeyValues = new HashSet(); + List documents = new List(); + for (int i = 0; i < numberOfDocuments; i++) + { + Document doc = new Document(); + doc.SetPropertyValue(partitionKey, i); + documents.Add(doc.ToString()); + + expectedPartitionKeyValues.Add(i); + } + + Assert.AreEqual(numberOfDocuments, expectedPartitionKeyValues.Count); + + QueryCrossPartitionWithLargeNumberOfKeysArgs args = new QueryCrossPartitionWithLargeNumberOfKeysArgs() + { + NumberOfDocuments = numberOfDocuments, + PartitionKey = partitionKey, + ExpectedPartitionKeyValues = expectedPartitionKeyValues, + }; + + await this.CreateIngestQueryDelete( + ConnectionModes.Direct | ConnectionModes.Gateway, + documents, + this.TestQueryCrossPartitionWithLargeNumberOfKeysHelper, + args, + "/" + partitionKey); + } + + private async Task TestQueryCrossPartitionWithLargeNumberOfKeysHelper(CosmosContainer container, IEnumerable documents, QueryCrossPartitionWithLargeNumberOfKeysArgs args) + { + CosmosSqlQueryDefinition query = new CosmosSqlQueryDefinition( + $"SELECT VALUE r.{args.PartitionKey} FROM r WHERE ARRAY_CONTAINS(@keys, r.{args.PartitionKey})").UseParameter("@keys", args.ExpectedPartitionKeyValues); + + HashSet actualPartitionKeyValues = new HashSet(); + FeedIterator documentQuery = container.Items.CreateItemQuery( + sqlQueryDefinition: query, + maxItemCount: -1, + maxConcurrency: 100); + + while (documentQuery.HasMoreResults) + { + FeedResponse response = await documentQuery.FetchNextSetAsync(); + foreach (int item in response) + { + actualPartitionKeyValues.Add(item); + } + } + + Assert.IsTrue(actualPartitionKeyValues.SetEquals(args.ExpectedPartitionKeyValues)); + } + + [TestMethod] + public async Task TestBasicCrossPartitionQuery() + { + int seed = (int)(DateTime.UtcNow - new DateTime(1970, 1, 1)).TotalSeconds; + uint numberOfDocuments = 100; + QueryOracle.QueryOracleUtil util = new QueryOracle.QueryOracle2(seed); + IEnumerable documents = util.GetDocuments(numberOfDocuments); + + await this.CreateIngestQueryDelete( + ConnectionModes.Direct, + documents, + this.TestBasicCrossPartitionQueryHelper); + } + + private async Task TestBasicCrossPartitionQueryHelper( + CosmosContainer container, + IEnumerable documents) + { + foreach (int maxDegreeOfParallelism in new int[] { 1, 100 }) + { + foreach (int maxItemCount in new int[] { 10, 100 }) + { + QueryRequestOptions feedOptions = new QueryRequestOptions + { + EnableCrossPartitionQuery = true, + MaxBufferedItemCount = 7000, + MaxConcurrency = maxDegreeOfParallelism + }; + + List actualFromQueryWithoutContinutionTokens; + actualFromQueryWithoutContinutionTokens = await QueryWithoutContinuationTokens( + container, + "SELECT * FROM c", + maxItemCount, + feedOptions); + + Assert.AreEqual(documents.Count(), actualFromQueryWithoutContinutionTokens.Count); + } + } + } + + [TestMethod] + public async Task TestQueryCrossPartitionAggregateFunctions() + { + AggregateTestArgs aggregateTestArgs = new AggregateTestArgs() + { + NumberOfDocsWithSamePartitionKey = 100, + NumberOfDocumentsDifferentPartitionKey = 100, + PartitionKey = "key", + UniquePartitionKey = "uniquePartitionKey", + Field = "field", + Values = new object[] { false, true, "abc", "cdfg", "opqrs", "ttttttt", "xyz" }, + }; + + List documents = new List(aggregateTestArgs.NumberOfDocumentsDifferentPartitionKey + aggregateTestArgs.NumberOfDocsWithSamePartitionKey); + foreach (object val in aggregateTestArgs.Values) + { + Document doc; + doc = new Document(); + doc.SetPropertyValue(aggregateTestArgs.PartitionKey, val); + doc.SetPropertyValue("id", Guid.NewGuid().ToString()); + + documents.Add(doc.ToString()); + } + + for (int i = 0; i < aggregateTestArgs.NumberOfDocsWithSamePartitionKey; ++i) + { + Document doc = new Document(); + doc.SetPropertyValue(aggregateTestArgs.PartitionKey, aggregateTestArgs.UniquePartitionKey); + doc.ResourceId = i.ToString(CultureInfo.InvariantCulture); + doc.SetPropertyValue(aggregateTestArgs.Field, i + 1); + doc.SetPropertyValue("id", Guid.NewGuid().ToString()); + + documents.Add(doc.ToString()); + } + + for (int i = 0; i < aggregateTestArgs.NumberOfDocumentsDifferentPartitionKey; ++i) + { + Document doc = new Document(); + doc.SetPropertyValue(aggregateTestArgs.PartitionKey, i + 1); + doc.SetPropertyValue("id", Guid.NewGuid().ToString()); + documents.Add(doc.ToString()); + } + + await this.CreateIngestQueryDelete( + ConnectionModes.Direct | ConnectionModes.Gateway, + documents, + this.TestQueryCrossPartitionAggregateFunctionsAsync, + aggregateTestArgs, + "/" + aggregateTestArgs.PartitionKey); + } + + private struct AggregateTestArgs + { + public int NumberOfDocumentsDifferentPartitionKey; + public int NumberOfDocsWithSamePartitionKey; + public string PartitionKey; + public string UniquePartitionKey; + public string Field; + public object[] Values; + } + + private struct AggregateQueryArguments + { + public string AggregateOperator; + public object ExpectedValue; + public string Predicate; + } + + private async Task TestQueryCrossPartitionAggregateFunctionsAsync(CosmosContainer container, IEnumerable documents, AggregateTestArgs aggregateTestArgs) + { + int numberOfDocumentsDifferentPartitionKey = aggregateTestArgs.NumberOfDocumentsDifferentPartitionKey; + int numberOfDocumentSamePartitionKey = aggregateTestArgs.NumberOfDocsWithSamePartitionKey; + int numberOfDocuments = aggregateTestArgs.NumberOfDocumentsDifferentPartitionKey + aggregateTestArgs.NumberOfDocsWithSamePartitionKey; + object[] values = aggregateTestArgs.Values; + string partitionKey = aggregateTestArgs.PartitionKey; + + double samePartitionSum = ((numberOfDocumentSamePartitionKey * (numberOfDocumentSamePartitionKey + 1)) / 2); + double differentPartitionSum = ((numberOfDocumentsDifferentPartitionKey * (numberOfDocumentsDifferentPartitionKey + 1)) / 2); + double partitionSum = samePartitionSum + differentPartitionSum; + AggregateQueryArguments[] aggregateQueryArgumentsList = new AggregateQueryArguments[] + { + new AggregateQueryArguments() + { + AggregateOperator = "AVG", + ExpectedValue = partitionSum / numberOfDocuments, + Predicate = $"IS_NUMBER(r.{partitionKey})", + }, + new AggregateQueryArguments() + { + AggregateOperator = "AVG", + ExpectedValue = Undefined.Value, + Predicate = "true", + }, + new AggregateQueryArguments() + { + AggregateOperator = "COUNT", + ExpectedValue = (long)numberOfDocuments + values.Length, + Predicate = "true", + }, + new AggregateQueryArguments() + { + AggregateOperator = "MAX", + ExpectedValue = "xyz", + Predicate = "true", + }, + new AggregateQueryArguments() + { + AggregateOperator = "MIN", + ExpectedValue = false, + Predicate = "true", + }, + new AggregateQueryArguments() + { + AggregateOperator = "SUM", + ExpectedValue = differentPartitionSum, + Predicate = $"IS_NUMBER(r.{partitionKey})", + }, + new AggregateQueryArguments() + { + AggregateOperator = "SUM", + ExpectedValue = Undefined.Value, + Predicate = $"true", + }, + }; + + foreach (int maxDoP in new[] { 0, 10 }) + { + foreach (AggregateQueryArguments argument in aggregateQueryArgumentsList) + { + string[] queryFormats = new[] + { + "SELECT VALUE {0}(r.{1}) FROM r WHERE {2}", + "SELECT VALUE {0}(r.{1}) FROM r WHERE {2} ORDER BY r.{1}" + }; + + foreach (string queryFormat in queryFormats) + { + string query = string.Format(CultureInfo.InvariantCulture, queryFormat, argument.AggregateOperator, partitionKey, argument.Predicate); + string message = string.Format(CultureInfo.InvariantCulture, "query: {0}, data: {1}", query, JsonConvert.SerializeObject(argument)); + List items = new List(); + + FeedIterator resultSetIterator = container.Items.CreateItemQuery( + query, + maxConcurrency: maxDoP); + while (resultSetIterator.HasMoreResults) + { + items.AddRange(await resultSetIterator.FetchNextSetAsync()); + } + + if (Undefined.Value.Equals(argument.ExpectedValue)) + { + Assert.AreEqual(0, items.Count, message); + } + else + { + object expected = argument.ExpectedValue; + object actual = items.Single(); + + if (expected is long) + { + expected = (double)((long)expected); + } + + if (actual is long) + { + actual = (double)((long)actual); + } + + Assert.AreEqual(expected, actual, message); + } + } + } + + // Single partition queries + double singlePartitionSum = samePartitionSum; + Tuple[] datum = new[] + { + Tuple.Create("AVG", singlePartitionSum / numberOfDocumentSamePartitionKey), + Tuple.Create("COUNT", (long)numberOfDocumentSamePartitionKey), + Tuple.Create("MAX", (long)numberOfDocumentSamePartitionKey), + Tuple.Create("MIN", (long)1), + Tuple.Create("SUM", (long)singlePartitionSum), + }; + + string field = aggregateTestArgs.Field; + string uniquePartitionKey = aggregateTestArgs.UniquePartitionKey; + foreach (Tuple data in datum) + { + string query = $"SELECT VALUE {data.Item1}(r.{field}) FROM r WHERE r.{partitionKey} = '{uniquePartitionKey}'"; + dynamic aggregate = (await QueryWithoutContinuationTokens(container, query, maxItemCount: 1)).Single(); + object expected = data.Item2; + + if (aggregate is long) + { + aggregate = (long)aggregate; + } + + if (expected is long) + { + expected = (long)expected; + } + + Assert.AreEqual( + expected, + aggregate, + string.Format(CultureInfo.InvariantCulture, "query: {0}, data: {1}", query, JsonConvert.SerializeObject(data))); + + // V3 doesn't support an equivalent to ToList() + // Aggregate queries need to be in the form SELECT VALUE + //query = $"SELECT {data.Item1}(r.{field}) FROM r WHERE r.{partitionKey} = '{uniquePartitionKey}'"; + //try + //{ + // documentClient.CreateDocumentQuery( + // collection, + // query).ToList().Single(); + // Assert.Fail($"Expect exception query: {query}"); + //} + //catch (AggregateException ex) + //{ + // if (!(ex.InnerException is CosmosException) || ((CosmosException)ex.InnerException).StatusCode != HttpStatusCode.BadRequest) + // { + // throw; + // } + //} + + // Make sure ExecuteNextAsync works for unsupported aggregate projection + FeedResponse page = await container.Items.CreateItemQuery(query, maxConcurrency: 1).FetchNextSetAsync(); + } + } + } + + [TestMethod] + public async Task TestQueryCrossPartitionAggregateFunctionsEmptyPartitions() + { + AggregateQueryEmptyPartitionsArgs args = new AggregateQueryEmptyPartitionsArgs() + { + NumDocuments = 100, + PartitionKey = "key", + UniqueField = "UniqueField", + }; + + List documents = new List(args.NumDocuments); + for (int i = 0; i < args.NumDocuments; ++i) + { + Document doc = new Document(); + doc.SetPropertyValue(args.PartitionKey, Guid.NewGuid()); + doc.SetPropertyValue(args.UniqueField, i); + documents.Add(doc.ToString()); + } + + await this.CreateIngestQueryDelete( + ConnectionModes.Direct | ConnectionModes.Gateway, + documents, + this.TestQueryCrossPartitionAggregateFunctionsEmptyPartitionsHelper, + args, + "/" + args.PartitionKey); + } + + private struct AggregateQueryEmptyPartitionsArgs + { + public int NumDocuments; + public string PartitionKey; + public string UniqueField; + } + + private async Task TestQueryCrossPartitionAggregateFunctionsEmptyPartitionsHelper(CosmosContainer container, IEnumerable documents, AggregateQueryEmptyPartitionsArgs args) + { + await CrossPartitionQueryTestsOnePartition.NoOp(); + int numDocuments = args.NumDocuments; + string partitionKey = args.PartitionKey; + string uniqueField = args.UniqueField; + + // Perform full fanouts but only match a single value that isn't the partition key. + // This leads to all other partitions returning { "" = UNDEFINDED, "count" = 0 } + // which should be ignored from the aggregation. + int valueOfInterest = args.NumDocuments / 2; + string[] queries = new string[] + { + $"SELECT VALUE AVG(c.{uniqueField}) FROM c WHERE c.{uniqueField} = {valueOfInterest}", + $"SELECT VALUE MIN(c.{uniqueField}) FROM c WHERE c.{uniqueField} = {valueOfInterest}", + $"SELECT VALUE MAX(c.{uniqueField}) FROM c WHERE c.{uniqueField} = {valueOfInterest}", + $"SELECT VALUE SUM(c.{uniqueField}) FROM c WHERE c.{uniqueField} = {valueOfInterest}", + }; + + foreach (string query in queries) + { + try + { + List items = await this.RunQuery( + container, + query, + maxConcurrency: 10); + + Assert.AreEqual(valueOfInterest, items.Single()); + } + catch (Exception ex) + { + Assert.Fail($"Something went wrong with query: {query}, ex: {ex}"); + } + } + } + + [TestMethod] + public async Task TestQueryCrossPartitionAggregateFunctionsWithMixedTypes() + { + AggregateQueryMixedTypes args = new AggregateQueryMixedTypes() + { + PartitionKey = "key", + Field = "field", + DoubleOnlyKey = "doubleOnly", + StringOnlyKey = "stringOnly", + BoolOnlyKey = "boolOnly", + NullOnlyKey = "nullOnly", + ObjectOnlyKey = "objectOnlyKey", + ArrayOnlyKey = "arrayOnlyKey", + OneObjectKey = "oneObjectKey", + OneArrayKey = "oneArrayKey", + UndefinedKey = "undefinedKey", + }; + + List documents = new List(); + Random random = new Random(1234); + for (int i = 0; i < 20; ++i) + { + Document doubleDoc = new Document(); + doubleDoc.SetPropertyValue(args.PartitionKey, Guid.NewGuid()); + doubleDoc.SetPropertyValue(args.Field, random.Next(1, 100000)); + documents.Add(doubleDoc.ToString()); + doubleDoc.SetPropertyValue(args.PartitionKey, args.DoubleOnlyKey); + documents.Add(doubleDoc.ToString()); + + Document stringDoc = new Document(); + stringDoc.SetPropertyValue(args.PartitionKey, Guid.NewGuid()); + stringDoc.SetPropertyValue(args.Field, random.NextDouble().ToString()); + documents.Add(stringDoc.ToString()); + stringDoc.SetPropertyValue(args.PartitionKey, args.StringOnlyKey); + documents.Add(stringDoc.ToString()); + + Document boolDoc = new Document(); + boolDoc.SetPropertyValue(args.PartitionKey, Guid.NewGuid()); + boolDoc.SetPropertyValue(args.Field, random.Next() % 2 == 0); + documents.Add(boolDoc.ToString()); + boolDoc.SetPropertyValue(args.PartitionKey, args.BoolOnlyKey); + documents.Add(boolDoc.ToString()); + + Document nullDoc = new Document(); + nullDoc.SetPropertyValue(args.PartitionKey, Guid.NewGuid()); + nullDoc.propertyBag.Add(args.Field, null); + documents.Add(nullDoc.ToString()); + nullDoc.SetPropertyValue(args.PartitionKey, args.NullOnlyKey); + documents.Add(nullDoc.ToString()); + + Document objectDoc = new Document(); + objectDoc.SetPropertyValue(args.PartitionKey, Guid.NewGuid()); + objectDoc.SetPropertyValue(args.Field, new object { }); + documents.Add(objectDoc.ToString()); + objectDoc.SetPropertyValue(args.PartitionKey, args.ObjectOnlyKey); + documents.Add(objectDoc.ToString()); + + Document arrayDoc = new Document(); + arrayDoc.SetPropertyValue(args.PartitionKey, Guid.NewGuid()); + arrayDoc.SetPropertyValue(args.Field, new object[] { }); + documents.Add(arrayDoc.ToString()); + arrayDoc.SetPropertyValue(args.PartitionKey, args.ArrayOnlyKey); + documents.Add(arrayDoc.ToString()); + } + + Document oneObjectDoc = new Document(); + oneObjectDoc.SetPropertyValue(args.PartitionKey, args.OneObjectKey); + oneObjectDoc.SetPropertyValue(args.Field, new object { }); + documents.Add(oneObjectDoc.ToString()); + + Document oneArrayDoc = new Document(); + oneArrayDoc.SetPropertyValue(args.PartitionKey, args.OneArrayKey); + oneArrayDoc.SetPropertyValue(args.Field, new object[] { }); + documents.Add(oneArrayDoc.ToString()); + + Document undefinedDoc = new Document(); + undefinedDoc.SetPropertyValue(args.PartitionKey, args.UndefinedKey); + // This doc does not have the field key set + documents.Add(undefinedDoc.ToString()); + + await this.CreateIngestQueryDelete( + ConnectionModes.Direct, + documents, + this.TestQueryCrossPartitionAggregateFunctionsWithMixedTypesHelper, + args, + "/" + args.PartitionKey); + } + + private struct AggregateQueryMixedTypes + { + public string PartitionKey; + public string Field; + public string DoubleOnlyKey; + public string StringOnlyKey; + public string BoolOnlyKey; + public string NullOnlyKey; + public string ObjectOnlyKey; + public string ArrayOnlyKey; + public string OneObjectKey; + public string OneArrayKey; + public string UndefinedKey; + } + + private async Task TestQueryCrossPartitionAggregateFunctionsWithMixedTypesHelper( + CosmosContainer container, + IEnumerable documents, + AggregateQueryMixedTypes args) + { + await CrossPartitionQueryTestsOnePartition.NoOp(); + string partitionKey = args.PartitionKey; + string field = args.Field; + string[] typeOnlyPartitionKeys = new string[] + { + args.DoubleOnlyKey, + args.StringOnlyKey, + args.BoolOnlyKey, + args.NullOnlyKey, + args.ObjectOnlyKey, + args.ArrayOnlyKey, + args.OneArrayKey, + args.OneObjectKey, + args.UndefinedKey + }; + + string[] aggregateOperators = new string[] { "AVG", "MIN", "MAX", "SUM", "COUNT" }; + string[] typeCheckFunctions = new string[] { "IS_ARRAY", "IS_BOOL", "IS_NULL", "IS_NUMBER", "IS_OBJECT", "IS_STRING", "IS_DEFINED", "IS_PRIMITIVE" }; + List queries = new List(); + foreach (string aggregateOperator in aggregateOperators) + { + foreach (string typeCheckFunction in typeCheckFunctions) + { + queries.Add( + $@" + SELECT VALUE {aggregateOperator} (c.{field}) + FROM c + WHERE {typeCheckFunction}(c.{field}) + "); + } + + foreach (string typeOnlyPartitionKey in typeOnlyPartitionKeys) + { + queries.Add( + $@" + SELECT VALUE {aggregateOperator} (c.{field}) + FROM c + WHERE c.{partitionKey} = ""{typeOnlyPartitionKey}"" + "); + } + }; + + // mixing primitive and non primitives + foreach (string minmaxop in new string[] { "MIN", "MAX" }) + { + foreach (string key in new string[] { args.OneObjectKey, args.OneArrayKey }) + { + queries.Add( + $@" + SELECT VALUE {minmaxop} (c.{field}) + FROM c + WHERE c.{partitionKey} IN (""{key}"", ""{args.DoubleOnlyKey}"") + "); + } + } + + + string filename = $"CrossPartitionQueryTests.AggregateMixedTypes"; + string outputPath = $"{filename}_output.xml"; + string baselinePath = $"{filename}_baseline.xml"; + XmlWriterSettings settings = new XmlWriterSettings() + { + OmitXmlDeclaration = true, + Indent = true, + NewLineOnAttributes = true, + }; + using (XmlWriter writer = XmlWriter.Create(outputPath, settings)) + { + writer.WriteStartDocument(); + writer.WriteStartElement("Results"); + foreach (string query in queries) + { + string formattedQuery = string.Join( + Environment.NewLine, + query.Trim().Split( + new[] { Environment.NewLine }, + StringSplitOptions.None) + .Select(x => x.Trim())); + + List items = await this.RunQuery( + container, + query, + 10, + null); + + writer.WriteStartElement("Result"); + writer.WriteStartElement("Query"); + writer.WriteCData(formattedQuery); + writer.WriteEndElement(); + writer.WriteStartElement("Aggregation"); + if (items.Count > 0) + { + writer.WriteCData(JsonConvert.SerializeObject(items.Single())); + } + writer.WriteEndElement(); + writer.WriteEndElement(); + } + writer.WriteEndElement(); + writer.WriteEndDocument(); + } + + Regex r = new Regex(">\\s+"); + string normalizedBaseline = r.Replace(File.ReadAllText(baselinePath), ">"); + string normalizedOutput = r.Replace(File.ReadAllText(outputPath), ">"); + + Assert.AreEqual(normalizedBaseline, normalizedOutput); + } + + [TestMethod] + public async Task TestQueryDistinct() + { + int seed = (int)(DateTime.UtcNow - new DateTime(1970, 1, 1)).TotalSeconds; + uint numberOfDocuments = 100; + + Random rand = new Random(seed); + List people = new List(); + + for (int i = 0; i < numberOfDocuments; i++) + { + Person person = CrossPartitionQueryTestsOnePartition.GetRandomPerson(rand); + for (int j = 0; j < rand.Next(0, 4); j++) + { + people.Add(person); + } + } + + List documents = new List(); + people = people.OrderBy((person) => Guid.NewGuid()).ToList(); + foreach (Person person in people) + { + documents.Add(JsonConvert.SerializeObject(person)); + } + + await this.CreateIngestQueryDelete( + ConnectionModes.Direct | ConnectionModes.Gateway, + documents, + this.TestQueryDistinct, + "/id"); + } + + private async Task TestQueryDistinct(CosmosContainer container, IEnumerable documents, dynamic testArgs = null) + { + #region Queries + // To verify distint queries you can run it once without the distinct clause and run it through a hash set + // then compare to the query with the distinct clause. + List queries = new List() + { + // basic distinct queries + "SELECT {0} VALUE null", + "SELECT {0} VALUE false", + "SELECT {0} VALUE true", + "SELECT {0} VALUE 1", + "SELECT {0} VALUE 'a'", + "SELECT {0} VALUE [null, true, false, 1, 'a']", + "SELECT {0} VALUE {{p1:null, p2:true, p3:false, p4:1, p5:'a'}}", + "SELECT {0} false AS p", + "SELECT {0} 1 AS p", + "SELECT {0} 'a' AS p", + "SELECT {0} [null, true, false, 1, 'a'] AS p", + "SELECT {0} {{p1:null, p2:true, p3:false, p4:1, p5:'a'}} AS p", + "SELECT {0} VALUE {{p1:null, p2:true, p3:false, p4:1, p5:'a'}}", + "SELECT {0} VALUE null FROM c", + "SELECT {0} VALUE false FROM c", + "SELECT {0} VALUE 1 FROM c", + "SELECT {0} VALUE 'a' FROM c", + "SELECT {0} VALUE [null, true, false, 1, 'a'] FROM c", + "SELECT {0} null AS p FROM c", + "SELECT {0} false AS p FROM c", + "SELECT {0} 1 AS p FROM c", + "SELECT {0} 'a' AS p FROM c", + "SELECT {0} [null, true, false, 1, 'a'] AS p FROM c", + "SELECT {0} {{p1:null, p2:true, p3:false, p4:1, p5:'a'}} AS p FROM c", + + // number value distinct queries + "SELECT {0} VALUE c.income from c", + "SELECT {0} VALUE c.age from c", + "SELECT {0} c.income, c.income AS income2 from c", + "SELECT {0} c.income, c.age from c", + "SELECT {0} VALUE [c.income, c.age] from c", + + // string value distinct queries + "SELECT {0} VALUE c.name from c", + "SELECT {0} VALUE c.city from c", + "SELECT {0} VALUE c.partitionKey from c", + "SELECT {0} c.name, c.name AS name2 from c", + "SELECT {0} c.name, c.city from c", + "SELECT {0} VALUE [c.name, c.city] from c", + + // array value distinct queries + "SELECT {0} VALUE c.children from c", + "SELECT {0} c.children, c.children AS children2 from c", + "SELECT {0} VALUE [c.name, c.age, c.pet] from c", + + // object value distinct queries + "SELECT {0} VALUE c.pet from c", + "SELECT {0} c.pet, c.pet AS pet2 from c", + + // scalar expressions distinct query + "SELECT {0} VALUE c.age % 2 FROM c", + "SELECT {0} VALUE ABS(c.age) FROM c", + "SELECT {0} VALUE LEFT(c.name, 1) FROM c", + "SELECT {0} VALUE c.name || ', ' || (c.city ?? '') FROM c", + "SELECT {0} VALUE ARRAY_LENGTH(c.children) FROM c", + "SELECT {0} VALUE IS_DEFINED(c.city) FROM c", + "SELECT {0} VALUE (c.children[0].age ?? 0) + (c.children[1].age ?? 0) FROM c", + + // distinct queries with order by + "SELECT {0} VALUE c.age FROM c ORDER BY c.age", + "SELECT {0} VALUE c.name FROM c ORDER BY c.name", + "SELECT {0} VALUE c.city FROM c ORDER BY c.city", + "SELECT {0} VALUE c.city FROM c ORDER BY c.age", + "SELECT {0} VALUE LEFT(c.name, 1) FROM c ORDER BY c.name", + + // distinct queries with top and no matching order by + "SELECT {0} TOP 2147483647 VALUE c.age FROM c", + + // distinct queries with top and matching order by + "SELECT {0} TOP 2147483647 VALUE c.age FROM c ORDER BY c.age", + + // distinct queries with aggregates + "SELECT {0} VALUE MAX(c.age) FROM c", + + // distinct queries with joins + "SELECT {0} VALUE c.age FROM p JOIN c IN p.children", + "SELECT {0} p.age AS ParentAge, c.age ChildAge FROM p JOIN c IN p.children", + "SELECT {0} VALUE c.name FROM p JOIN c IN p.children", + "SELECT {0} p.name AS ParentName, c.name ChildName FROM p JOIN c IN p.children", + + // distinct queries in subqueries + "SELECT {0} r.age, s FROM r JOIN (SELECT DISTINCT VALUE c FROM (SELECT 1 a) c) s WHERE r.age > 25", + "SELECT {0} p.name, p.age FROM (SELECT DISTINCT * FROM r) p WHERE p.age > 25", + + // distinct queries in scalar subqeries + "SELECT {0} p.name, (SELECT DISTINCT VALUE p.age) AS Age FROM p", + "SELECT {0} p.name, p.age FROM p WHERE (SELECT DISTINCT VALUE LEFT(p.name, 1)) > 'A' AND (SELECT DISTINCT VALUE p.age) > 21", + "SELECT {0} p.name, (SELECT DISTINCT VALUE p.age) AS Age FROM p WHERE (SELECT DISTINCT VALUE p.name) > 'A' OR (SELECT DISTINCT VALUE p.age) > 21", + + // select * + "SELECT {0} * FROM c", + }; + #endregion + #region ExecuteNextAsync API + // run the query with distinct and without + MockDistinctMap + // Should receive same results + // PageSize = 1 guarantees that the backend will return some duplicates. + foreach (string query in queries) + { + foreach (int pageSize in new int[] { 1, 10, 100 }) + { + string queryWithDistinct = string.Format(query, "DISTINCT"); + string queryWithoutDistinct = string.Format(query, ""); + MockDistinctMap documentsSeen = new MockDistinctMap(); + List documentsFromWithDistinct = new List(); + List documentsFromWithoutDistinct = new List(); + + FeedIterator documentQueryWithoutDistinct = container.Items.CreateItemQuery( + queryWithoutDistinct, + maxConcurrency: 100, + maxItemCount: pageSize); + + while (documentQueryWithoutDistinct.HasMoreResults) + { + FeedResponse FeedResponse = await documentQueryWithoutDistinct.FetchNextSetAsync(); + foreach (JToken document in FeedResponse) + { + if (documentsSeen.Add(document, out UInt192? hash)) + { + documentsFromWithoutDistinct.Add(document); + } + else + { + // No Op for debugging purposes. + } + } + } + + FeedIterator documentQueryWithDistinct = container.Items.CreateItemQuery( + queryWithDistinct, + maxConcurrency: 100, + maxItemCount: pageSize); + + while (documentQueryWithDistinct.HasMoreResults) + { + FeedResponse FeedResponse = await documentQueryWithDistinct.FetchNextSetAsync(); + documentsFromWithDistinct.AddRange(FeedResponse); + } + + try + { + Assert.AreEqual(documentsFromWithDistinct.Count, documentsFromWithoutDistinct.Count()); + for (int i = 0; i < documentsFromWithDistinct.Count; i++) + { + JToken documentFromWithDistinct = documentsFromWithDistinct.ElementAt(i); + JToken documentFromWithoutDistinct = documentsFromWithoutDistinct.ElementAt(i); + Assert.IsTrue( + JsonTokenEqualityComparer.Value.Equals(documentFromWithDistinct, documentFromWithoutDistinct), + $"{documentFromWithDistinct} did not match {documentFromWithoutDistinct} at index {i} for {queryWithDistinct}, with page size: {pageSize} on a container"); + } + } + catch (Exception e) + { + throw e; + } + } + } + #endregion + #region Unordered Continuation + // Run the unordered distinct query through the continuation api should result in the same set(but maybe some duplicates) + foreach (string query in new string[] + { + "SELECT {0} VALUE c.name from c", + "SELECT {0} VALUE c.age from c", + "SELECT {0} TOP 2147483647 VALUE c.city from c", + "SELECT {0} VALUE c.age from c ORDER BY c.name", + }) + { + string queryWithDistinct = string.Format(query, "DISTINCT"); + string queryWithoutDistinct = string.Format(query, ""); + HashSet documentsFromWithDistinct = new HashSet(JsonTokenEqualityComparer.Value); + HashSet documentsFromWithoutDistinct = new HashSet(JsonTokenEqualityComparer.Value); + + FeedIterator documentQueryWithoutDistinct = container.Items.CreateItemQuery( + queryWithoutDistinct, + maxItemCount: 10, + maxConcurrency: 100); + + while (documentQueryWithoutDistinct.HasMoreResults) + { + FeedResponse FeedResponse = await documentQueryWithoutDistinct.FetchNextSetAsync(); + foreach (JToken jToken in FeedResponse) + { + documentsFromWithoutDistinct.Add(jToken); + } + } + + FeedIterator documentQueryWithDistinct = container.Items.CreateItemQuery( + queryWithDistinct, + maxItemCount: 10, + maxConcurrency: 100); + + // For now we are blocking the use of continuation + // This try catch can be removed if we do allow the continuation token. + try + { + string continuationToken = null; + do + { + FeedIterator documentQuery = container.Items.CreateItemQuery( + queryWithDistinct, + maxItemCount: 10, + maxConcurrency: 100); + + FeedResponse FeedResponse = await documentQuery.FetchNextSetAsync(); + foreach (JToken jToken in FeedResponse) + { + documentsFromWithDistinct.Add(jToken); + } + + continuationToken = FeedResponse.Continuation; + + } + while (continuationToken != null); + Assert.IsTrue( + documentsFromWithDistinct.IsSubsetOf(documentsFromWithoutDistinct), + $"Documents didn't match for {queryWithDistinct} on a Partitioned container"); + + Assert.Fail("Expected an exception when using continuation tokens on an unordered distinct query."); + } + catch (ArgumentException ex) + { + string disallowContinuationErrorMessage = RMResources.UnorderedDistinctQueryContinuationToken; + Assert.AreEqual(disallowContinuationErrorMessage, ex.Message); + } + } + #endregion + #region Ordered Region + // Run the ordered distinct query through the continuation api, should result in the same set + // since the previous hash is passed in the continuation token. + foreach (string query in new string[] + { + "SELECT {0} VALUE c.age FROM c ORDER BY c.age", + "SELECT {0} VALUE c.name FROM c ORDER BY c.name", + }) + { + foreach (int pageSize in new int[] { 1, 10, 100 }) + { + string queryWithDistinct = string.Format(query, "DISTINCT"); + string queryWithoutDistinct = string.Format(query, ""); + MockDistinctMap documentsSeen = new MockDistinctMap(); + List documentsFromWithDistinct = new List(); + List documentsFromWithoutDistinct = new List(); + + FeedIterator documentQueryWithoutDistinct = container.Items.CreateItemQuery( + sqlQueryText: queryWithoutDistinct, + maxConcurrency: 100, + maxItemCount: 1); + + while (documentQueryWithoutDistinct.HasMoreResults) + { + FeedResponse FeedResponse = await documentQueryWithoutDistinct.FetchNextSetAsync(); + foreach (JToken document in FeedResponse) + { + if (documentsSeen.Add(document, out UInt192? hash)) + { + documentsFromWithoutDistinct.Add(document); + } + else + { + // No Op for debugging purposes. + } + } + } + + FeedIterator documentQueryWithDistinct = container.Items.CreateItemQuery( + sqlQueryText: queryWithDistinct, + maxConcurrency: 100, + maxItemCount: 1); + + string continuationToken = null; + do + { + FeedIterator cosmosQuery = container.Items.CreateItemQuery( + sqlQueryText: queryWithDistinct, + maxConcurrency: 100, + maxItemCount: 1, + continuationToken: continuationToken); + + FeedResponse FeedResponse = await cosmosQuery.FetchNextSetAsync(); + documentsFromWithDistinct.AddRange(FeedResponse); + continuationToken = FeedResponse.Continuation; + } + while (continuationToken != null); + + Assert.IsTrue( + documentsFromWithDistinct.SequenceEqual(documentsFromWithoutDistinct, JsonTokenEqualityComparer.Value), + $"Documents didn't match for {queryWithDistinct} on a Partitioned container"); + } + } + #endregion + } + + [TestMethod] + public async Task TestQueryCrossPartitionTopOrderByDifferentDimension() + { + string[] documents = new[] + { + @"{""id"":""documentId1"",""key"":""A""}", + @"{""id"":""documentId2"",""key"":""A"",""prop"":3}", + @"{""id"":""documentId3"",""key"":""A""}", + @"{""id"":""documentId4"",""key"":5}", + @"{""id"":""documentId5"",""key"":5,""prop"":2}", + @"{""id"":""documentId6"",""key"":5}", + @"{""id"":""documentId7"",""key"":2}", + @"{""id"":""documentId8"",""key"":2,""prop"":1}", + @"{""id"":""documentId9"",""key"":2}", + }; + + await this.CreateIngestQueryDelete( + ConnectionModes.Direct | ConnectionModes.Gateway, + documents, + this.TestQueryCrossPartitionTopOrderByDifferentDimensionHelper, + "/key"); + } + + private async Task TestQueryCrossPartitionTopOrderByDifferentDimensionHelper(CosmosContainer container, IEnumerable documents) + { + await CrossPartitionQueryTestsOnePartition.NoOp(); + + string[] expected = new[] { "documentId2", "documentId5", "documentId8" }; + List query = await this.RunQuery( + container, + "SELECT r.id FROM r ORDER BY r.prop DESC", + maxItemCount: 1, + maxConcurrency: 1); + + Assert.AreEqual(string.Join(", ", expected), string.Join(", ", query.Select(doc => doc.Id))); + } + + [TestMethod] + public async Task TestOrderByNonAsciiCharacters() + { + string[] specialStrings = new string[] + { + // Strings which may be used elsewhere in code + "undefined", + // Numeric Strings + "-9223372036854775808/-1", + // Non-whitespace C0 controls: U+0001 through U+0008, U+000E through U+001F, + "\u0001", + // "Byte order marks" + "U+FEFF", + // Unicode Symbols + "ЁЂЃЄЅІЇЈЉЊЋЌЍЎЏАБВГДЕЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯабвгдежзийклмнопрстуфхцчшщъыьэюя", + // Quotation Marks + "", + // Strings which contain two-byte characters: can cause rendering issues or character-length issues + "찦차를 타고 온 펲시맨과 쑛다리 똠방각하", + // Changing length when lowercased + "Ⱥ", + // Japanese Emoticons + "ﾟ･✿ヾ╲(｡◕‿◕｡)╱✿･ﾟ", + // Emoji + "❤️ 💔 💌 💕 💞 💓 💗 💖 💘 💝 💟 💜 💛 💚 💙", + // Strings which contain "corrupted" text. The corruption will not appear in non-HTML text, however. (via http://www.eeemo.net) + "Ṱ̺̺̕o͞ ̷i̲̬͇̪͙n̝̗͕v̟̜̘̦͟o̶̙̰̠kè͚̮̺̪̹̱̤ ̖t̝͕̳̣̻̪͞h̼͓̲̦̳̘̲e͇̣̰̦̬͎ ̢̼̻̱̘h͚͎͙̜̣̲ͅi̦̲̣̰̤v̻͍e̺̭̳̪̰-m̢iͅn̖̺̞̲̯̰d̵̼̟͙̩̼̘̳ ̞̥̱̳̭r̛̗̘e͙p͠r̼̞̻̭̗e̺̠̣͟s̘͇̳͍̝͉e͉̥̯̞̲͚̬͜ǹ̬͎͎̟̖͇̤t͍̬̤͓̼̭͘ͅi̪̱n͠g̴͉ ͏͉ͅc̬̟h͡a̫̻̯͘o̫̟̖͍̙̝͉s̗̦̲.̨̹͈̣" + + }; + + IEnumerable documents = specialStrings.Select((specialString) => $@"{{ ""field"" : ""{specialString}""}}"); + await this.CreateIngestQueryDelete( + ConnectionModes.Direct | ConnectionModes.Gateway, + documents, + this.TestOrderByNonAsciiCharactersHelper); + } + + private async Task TestOrderByNonAsciiCharactersHelper( + CosmosContainer container, + IEnumerable documents) + { + foreach (int maxDegreeOfParallelism in new int[] { 1, 100 }) + { + foreach (int maxItemCount in new int[] { 10, 100 }) + { + QueryRequestOptions feedOptions = new QueryRequestOptions + { + EnableCrossPartitionQuery = true, + MaxBufferedItemCount = 7000, + MaxConcurrency = maxDegreeOfParallelism + }; + + List actualFromQueryWithoutContinutionTokens = await QueryWithContinuationTokens( + container, + "SELECT * FROM c ORDER BY c.field", + maxItemCount, + feedOptions); + + Assert.AreEqual(documents.Count(), actualFromQueryWithoutContinutionTokens.Count); + } + } + } + + [TestMethod] + public async Task TestMixedTypeOrderBy() + { + int numberOfDocuments = 1 << 4; + int numberOfDuplicates = 1 << 2; + + List documents = new List(numberOfDocuments * numberOfDuplicates); + Random random = new Random(1234); + for (int i = 0; i < numberOfDocuments; ++i) + { + MixedTypedDocument mixedTypeDocument = CrossPartitionQueryTestsOnePartition.GenerateMixedTypeDocument(random); + for (int j = 0; j < numberOfDuplicates; j++) + { + documents.Add(JsonConvert.SerializeObject(mixedTypeDocument)); ; + } + } + + // Just have range indexes + Cosmos.IndexingPolicy indexV1Policy = new Cosmos.IndexingPolicy() + { + IncludedPaths = new Collection() + { + new Cosmos.IncludedPath() + { + Path = "/*", + Indexes = new Collection() + { + Cosmos.Index.Range(Cosmos.DataType.String, -1), + Cosmos.Index.Range(Cosmos.DataType.Number, -1), + } + } + } + }; + + // Add a composite index to force an index v2 container to be made. + Cosmos.IndexingPolicy indexV2Policy = new Cosmos.IndexingPolicy() + { + IncludedPaths = new Collection() + { + new Cosmos.IncludedPath() + { + Path = "/*", + } + }, + + CompositeIndexes = new Collection>() + { + // Simple + new Collection() + { + new Cosmos.CompositePath() + { + Path = "/_ts", + }, + new Cosmos.CompositePath() + { + Path = "/_etag", + } + } + } + }; + + string indexV2Api = HttpConstants.Versions.v2018_09_17; + string indexV1Api = HttpConstants.Versions.v2017_11_15; + + Func, Task> runWithAllowMixedTypeOrderByFlag = async (allowMixedTypeOrderByTestFlag, orderByTypes, expectedExcpetionHandler) => + { + bool allowMixedTypeOrderByTestFlagOriginalValue = OrderByConsumeComparer.AllowMixedTypeOrderByTestFlag; + string apiVersion = allowMixedTypeOrderByTestFlag ? indexV2Api : indexV1Api; + Cosmos.IndexingPolicy indexingPolicy = allowMixedTypeOrderByTestFlag ? indexV2Policy : indexV1Policy; + try + { + OrderByConsumeComparer.AllowMixedTypeOrderByTestFlag = allowMixedTypeOrderByTestFlag; + await this.RunWithApiVersion( + apiVersion, + async () => + { + await this.CreateIngestQueryDelete>>( + ConnectionModes.Direct, + documents, + this.TestMixedTypeOrderByHelper, + new Tuple>(orderByTypes, expectedExcpetionHandler), + "/id", + indexingPolicy); + }); + } + finally + { + OrderByConsumeComparer.AllowMixedTypeOrderByTestFlag = allowMixedTypeOrderByTestFlagOriginalValue; + } + }; + + bool dontAllowMixedTypes = false; + bool doAllowMixedTypes = true; + + OrderByTypes primitives = OrderByTypes.Bool | OrderByTypes.Null | OrderByTypes.Number | OrderByTypes.String; + OrderByTypes nonPrimitives = OrderByTypes.Array | OrderByTypes.Object; + OrderByTypes all = primitives | nonPrimitives | OrderByTypes.Undefined; + + // Don't allow mixed types but single type order by should still work + await runWithAllowMixedTypeOrderByFlag( + dontAllowMixedTypes, + new OrderByTypes[] + { + OrderByTypes.Array, + OrderByTypes.Bool, + OrderByTypes.Null, + OrderByTypes.Number, + OrderByTypes.Object, + OrderByTypes.String, + OrderByTypes.Undefined, + }, null); + + // If you don't allow mixed types but you run a mixed type query then you should get an exception or the results are just wrong. + await runWithAllowMixedTypeOrderByFlag( + dontAllowMixedTypes, + new OrderByTypes[] + { + all, + primitives, + }, + (exception) => + { + Assert.IsTrue( + // Either we get the weird client exception for having mixed types + exception.Message.Contains("Cannot execute cross partition order-by queries on mix types.") + // Or the results are just messed up since the pages in isolation were not mixed typed. + || exception.GetType() == typeof(AssertFailedException)); + }); + + // Mixed type orderby should work for all scenarios, + // since for now the non primitives are accepted to not be served from the index. + await runWithAllowMixedTypeOrderByFlag( + doAllowMixedTypes, + new OrderByTypes[] + { + OrderByTypes.Array, + OrderByTypes.Bool, + OrderByTypes.Null, + OrderByTypes.Number, + OrderByTypes.Object, + OrderByTypes.String, + OrderByTypes.Undefined, + primitives, + nonPrimitives, + all, + }, null); + } + + private sealed class MixedTypedDocument + { + public object MixedTypeField { get; set; } + } + + private static MixedTypedDocument GenerateMixedTypeDocument(Random random) + { + return new MixedTypedDocument() + { + MixedTypeField = GenerateRandomJsonValue(random), + }; + } + + private static object GenerateRandomJsonValue(Random random) + { + switch (random.Next(0, 6)) + { + // Number + case 0: + return random.Next(); + // String + case 1: + return new string('a', random.Next(0, 100)); + // Null + case 2: + return null; + // Bool + case 3: + return (random.Next() % 2) == 0; + // Object + case 4: + return new object(); + // Array + case 5: + return new List(); + default: + throw new ArgumentException(); + } + } + + private sealed class MockOrderByComparer : IComparer + { + public static readonly MockOrderByComparer Value = new MockOrderByComparer(); + + public int Compare(object x, object y) + { + CosmosElement element1 = ObjectToCosmosElement(x); + CosmosElement element2 = ObjectToCosmosElement(y); + + return ItemComparer.Instance.Compare(element1, element2); + } + + private static CosmosElement ObjectToCosmosElement(object obj) + { + string json = JsonConvert.SerializeObject(obj != null ? JToken.FromObject(obj) : JValue.CreateNull()); + byte[] bytes = Encoding.UTF8.GetBytes(json); + return CosmosElement.Create(bytes); + } + } + + [Flags] + private enum OrderByTypes + { + Number = 1 << 0, + String = 1 << 1, + Null = 1 << 2, + Bool = 1 << 3, + Object = 1 << 4, + Array = 1 << 5, + Undefined = 1 << 6, + }; + + private async Task TestMixedTypeOrderByHelper( + CosmosContainer container, + IEnumerable documents, + Tuple> args) + { + OrderByTypes[] orderByTypesList = args.Item1; + Action expectedExceptionHandler = args.Item2; + try + { + foreach (bool isDesc in new bool[] { true, false }) + { + foreach (OrderByTypes orderByTypes in orderByTypesList) + { + string orderString = isDesc ? "DESC" : "ASC"; + List mixedTypeFilters = new List(); + if (orderByTypes.HasFlag(OrderByTypes.Array)) + { + mixedTypeFilters.Add($"IS_ARRAY(c.{nameof(MixedTypedDocument.MixedTypeField)})"); + } + + if (orderByTypes.HasFlag(OrderByTypes.Bool)) + { + mixedTypeFilters.Add($"IS_BOOL(c.{nameof(MixedTypedDocument.MixedTypeField)})"); + } + + if (orderByTypes.HasFlag(OrderByTypes.Null)) + { + mixedTypeFilters.Add($"IS_NULL(c.{nameof(MixedTypedDocument.MixedTypeField)})"); + } + + if (orderByTypes.HasFlag(OrderByTypes.Number)) + { + mixedTypeFilters.Add($"IS_NUMBER(c.{nameof(MixedTypedDocument.MixedTypeField)})"); + } + + if (orderByTypes.HasFlag(OrderByTypes.Object)) + { + mixedTypeFilters.Add($"IS_OBJECT(c.{nameof(MixedTypedDocument.MixedTypeField)})"); + } + + if (orderByTypes.HasFlag(OrderByTypes.String)) + { + mixedTypeFilters.Add($"IS_STRING(c.{nameof(MixedTypedDocument.MixedTypeField)})"); + } + + if (orderByTypes.HasFlag(OrderByTypes.Undefined)) + { + mixedTypeFilters.Add($"not IS_DEFINED(c.{nameof(MixedTypedDocument.MixedTypeField)})"); + } + + string filter = mixedTypeFilters.Count() == 0 ? "true" : string.Join(" OR ", mixedTypeFilters); + + string query = $@" + SELECT VALUE c.{nameof(MixedTypedDocument.MixedTypeField)} + FROM c + WHERE {filter} + ORDER BY c.{nameof(MixedTypedDocument.MixedTypeField)} {orderString}"; + + QueryRequestOptions feedOptions = new QueryRequestOptions() + { + MaxBufferedItemCount = 1000, + }; + + List actualFromQueryWithoutContinutionTokens; + actualFromQueryWithoutContinutionTokens = await this.RunQuery( + container, + query, + maxItemCount: 16, + maxConcurrency: 10, + requestOptions: feedOptions); +#if false + For now we can not serve the query through continuation tokens correctly. + This is because we allow order by on mixed types but not comparisions across types + For example suppose the following query: + SELECT c.MixedTypeField FROM c ORDER BY c.MixedTypeField + returns: + [ + {"MixedTypeField":null}, + {"MixedTypeField":false}, + {"MixedTypeField":true}, + {"MixedTypeField":303093052}, + {"MixedTypeField":438985130}, + {"MixedTypeField":"aaaaaaaaaaa"} + ] + and we left off on 303093052 then at some point the cross partition code resumes the query by running the following: + SELECT c.MixedTypeField FROM c WHERE c.MixedTypeField > 303093052 ORDER BY c.MixedTypeField + which will only return the following: + { "MixedTypeField":438985130} + and that is because comparision across types is undefined so "aaaaaaaaaaa" > 303093052 never got emitted +#endif + + IEnumerable insertedDocs = documents + .Select(document => document.GetPropertyValue(nameof(MixedTypedDocument.MixedTypeField))); + + // Build the expected results using LINQ + IEnumerable expected = new List(); + + // Filter based on the mixedOrderByType enum + if (orderByTypes.HasFlag(OrderByTypes.Array)) + { + // no arrays should be served from the range index + } + + if (orderByTypes.HasFlag(OrderByTypes.Bool)) + { + expected = expected.Concat(insertedDocs.Where(x => x is bool)); + } + + if (orderByTypes.HasFlag(OrderByTypes.Null)) + { + expected = expected.Concat(insertedDocs.Where(x => x == null)); + } + + if (orderByTypes.HasFlag(OrderByTypes.Number)) + { + expected = expected.Concat(insertedDocs.Where(x => x is double || x is int || x is long)); + } + + if (orderByTypes.HasFlag(OrderByTypes.Object)) + { + // no objects should be served from the range index + } + + if (orderByTypes.HasFlag(OrderByTypes.String)) + { + expected = expected.Concat(insertedDocs.Where(x => x is string)); + } + + if (orderByTypes.HasFlag(OrderByTypes.Undefined)) + { + // no undefined should be served from the range index + } + + // Order using the mock order by comparer + if (isDesc) + { + expected = expected.OrderByDescending(x => x, MockOrderByComparer.Value); + } + else + { + expected = expected.OrderBy(x => x, MockOrderByComparer.Value); + } + + // bind all the value to JTokens so they can be compared agaisnt the actual. + List expectedBinded = expected.Select(x => x == null ? JValue.CreateNull() : JToken.FromObject(x)).ToList(); + + Assert.IsTrue( + expectedBinded.SequenceEqual(actualFromQueryWithoutContinutionTokens, JsonTokenEqualityComparer.Value), + $@" queryWithoutContinuations: {query}, + expected:{JsonConvert.SerializeObject(expected)}, + actual: {JsonConvert.SerializeObject(actualFromQueryWithoutContinutionTokens)}"); + + // Can't assert for reasons mentioned above + //Assert.IsTrue( + // expected.SequenceEqual(actualFromQueryWithContinutionTokens, DistinctMapTests.JsonTokenEqualityComparer.Value), + // $@" queryWithContinuations: {query}, + // expected:{JsonConvert.SerializeObject(expected)}, + // actual: {JsonConvert.SerializeObject(actualFromQueryWithContinutionTokens)}"); + } + } + } + catch (Exception ex) + { + if (expectedExceptionHandler != null) + { + expectedExceptionHandler(ex); + } + else + { + throw; + } + } + } + + [TestMethod] + public async Task TestQueryCrossPartitionTopOrderBy() + { + int seed = (int)(DateTime.UtcNow - new DateTime(1970, 1, 1)).TotalSeconds; + uint numberOfDocuments = 1000; + string partitionKey = "field_0"; + + QueryOracle.QueryOracleUtil util = new QueryOracle.QueryOracle2(seed); + IEnumerable documents = util.GetDocuments(numberOfDocuments); + + await this.CreateIngestQueryDelete( + ConnectionModes.Direct | ConnectionModes.Gateway, + documents, + this.TestQueryCrossPartitionTopOrderByHelper, + partitionKey, + "/" + partitionKey); + } + + private async Task TestQueryCrossPartitionTopOrderByHelper(CosmosContainer container, IEnumerable documents, string testArg) + { + string partitionKey = testArg; + IDictionary idToRangeMinKeyMap = new Dictionary(); + IRoutingMapProvider routingMapProvider = await this.Client.DocumentClient.GetPartitionKeyRangeCacheAsync(); + + CosmosContainerSettings containerSettings = await container.ReadAsync(); + foreach (Document document in documents) + { + IReadOnlyList targetRanges = await routingMapProvider.TryGetOverlappingRangesAsync( + containerSettings.ResourceId, + Range.GetPointRange( + PartitionKeyInternal.FromObjectArray( + new object[] + { + document.GetValue(partitionKey) + }, + true).GetEffectivePartitionKeyString(containerSettings.PartitionKey))); + Debug.Assert(targetRanges.Count == 1); + idToRangeMinKeyMap.Add(document.Id, targetRanges[0].MinInclusive); + } + + IList partitionKeyValues = new HashSet(documents.Select(doc => doc.GetValue(partitionKey))).ToList(); + + // Test Empty Results + List expectedResults = new List { }; + List computedResults = new List(); + + string emptyQueryText = @"SELECT TOP 5 * FROM Root r WHERE r.partitionKey = 9991123 OR r.partitionKey = 9991124 OR r.partitionKey = 99991125"; + FeedOptions feedOptionsEmptyResult = new FeedOptions + { + EnableCrossPartitionQuery = true + }; + + List queryEmptyResult = await this.RunQuery( + container, + emptyQueryText, + maxConcurrency: 1); + + computedResults = queryEmptyResult.Select(doc => doc.Id).ToList(); + computedResults.Sort(); + expectedResults.Sort(); + + Random rand = new Random(); + Assert.AreEqual(string.Join(",", expectedResults), string.Join(",", computedResults)); + List tasks = new List(); + for (int trial = 0; trial < 1; ++trial) + { + foreach (bool fanOut in new[] { true, false }) + { + foreach (bool isParametrized in new[] { true, false }) + { + foreach (bool hasTop in new[] { false, true }) + { + foreach (bool hasOrderBy in new[] { false, true }) + { + foreach (string sortOrder in new[] { string.Empty, "ASC", "DESC" }) + { + #region Expected Documents + string topValueName = "@topValue"; + int top = rand.Next(4) * rand.Next(partitionKeyValues.Count); + string queryText; + string orderByField = "field_" + rand.Next(10); + IEnumerable filteredDocuments; + + Func getTop = () => + hasTop ? string.Format(CultureInfo.InvariantCulture, "TOP {0} ", isParametrized ? topValueName : top.ToString()) : string.Empty; + + Func getOrderBy = () => + hasOrderBy ? string.Format(CultureInfo.InvariantCulture, " ORDER BY r.{0} {1}", orderByField, sortOrder) : string.Empty; + + if (fanOut) + { + queryText = string.Format( + CultureInfo.InvariantCulture, + "SELECT {0}r.id, r.{1} FROM r{2}", + getTop(), + partitionKey, + getOrderBy()); + + filteredDocuments = documents; + } + else + { + HashSet selectedPartitionKeyValues = new HashSet(partitionKeyValues + .OrderBy(x => rand.Next()) + .ThenBy(x => x) + .Take(rand.Next(1, Math.Min(100, partitionKeyValues.Count) + 1))); + + queryText = string.Format( + CultureInfo.InvariantCulture, + "SELECT {0}r.id, r.{1} FROM r WHERE r.{2} IN ({3}){4}", + getTop(), + partitionKey, + partitionKey, + string.Join(", ", selectedPartitionKeyValues), + getOrderBy()); + + filteredDocuments = documents + .AsParallel() + .Where(doc => selectedPartitionKeyValues.Contains(doc.GetValue(partitionKey))); + } + + if (hasOrderBy) + { + switch (sortOrder) + { + case "": + case "ASC": + filteredDocuments = filteredDocuments + .AsParallel() + .OrderBy(doc => doc.GetValue(orderByField)) + .ThenBy(doc => idToRangeMinKeyMap[doc.Id]) + .ThenBy(doc => int.Parse(doc.Id, CultureInfo.InvariantCulture)); + break; + case "DESC": + filteredDocuments = filteredDocuments + .AsParallel() + .OrderByDescending(doc => doc.GetValue(orderByField)) + .ThenBy(doc => idToRangeMinKeyMap[doc.Id]) + .ThenByDescending(doc => int.Parse(doc.Id, CultureInfo.InvariantCulture)); + break; + } + } + else + { + filteredDocuments = filteredDocuments + .AsParallel() + .OrderBy(doc => idToRangeMinKeyMap[doc.Id]) + .ThenBy(doc => int.Parse(doc.Id, CultureInfo.InvariantCulture)); + } + + if (hasTop) + { + filteredDocuments = filteredDocuments.Take(top); + } + #endregion + #region Actual Documents + IEnumerable actualDocuments; + + int maxDegreeOfParallelism = hasTop ? rand.Next(4) : (rand.Next(2) == 0 ? -1 : (1 + rand.Next(0, 10))); + int? maxItemCount = rand.Next(2) == 0 ? -1 : rand.Next(1, documents.Count()); + QueryRequestOptions feedOptions = new QueryRequestOptions + { + MaxBufferedItemCount = rand.Next(2) == 0 ? -1 : rand.Next(Math.Min(100, documents.Count()), documents.Count() + 1), + }; + + if (rand.Next(3) == 0) + { + maxItemCount = null; + } + + CosmosSqlQueryDefinition querySpec = new CosmosSqlQueryDefinition(queryText); + SqlParameterCollection parameters = new SqlParameterCollection(); + if (isParametrized) + { + if (hasTop) + { + querySpec.UseParameter(topValueName, top); + } + } + + DateTime startTime = DateTime.Now; + List result = new List(); + FeedIterator query = container.Items.CreateItemQuery( + querySpec, + maxConcurrency: maxDegreeOfParallelism, + requestOptions: feedOptions); + + while (query.HasMoreResults) + { + FeedResponse response = await query.FetchNextSetAsync(); + result.AddRange(response); + } + + actualDocuments = result; + + #endregion + + double time = (DateTime.Now - startTime).TotalMilliseconds; + + Trace.TraceInformation(": {0}, : {1}, : {2}, : {3}, : {4},