diff --git a/Microsoft.Azure.Cosmos/src/CosmosClient.cs b/Microsoft.Azure.Cosmos/src/CosmosClient.cs index bf79ba3532..ff5c226f2b 100644 --- a/Microsoft.Azure.Cosmos/src/CosmosClient.cs +++ b/Microsoft.Azure.Cosmos/src/CosmosClient.cs @@ -5,11 +5,12 @@ namespace Microsoft.Azure.Cosmos { using System; + using System.Text; using System.Threading; using System.Threading.Tasks; + using Microsoft.Azure.Cosmos.Handlers; using Microsoft.Azure.Cosmos.Query; using Microsoft.Azure.Documents; - using System.Text; /// /// Provides a client-side logical representation of the Azure Cosmos DB database account. @@ -75,6 +76,10 @@ static CosmosClient() { HttpConstants.Versions.CurrentVersion = HttpConstants.Versions.v2018_12_31; HttpConstants.Versions.CurrentVersionUTF8 = Encoding.UTF8.GetBytes(HttpConstants.Versions.CurrentVersion); + + // V3 always assumes assemblies exists + // Shall revisit on feedback + ServiceInteropWrapper.AssembliesExist = new Lazy(() => true); } /// @@ -202,7 +207,7 @@ internal CosmosClient( internal CosmosOffers Offers => this.offerSet.Value; internal DocumentClient DocumentClient { get; set; } - internal CosmosRequestHandler RequestHandler { get; private set; } + internal RequestInvokerHandler RequestHandler { get; private set; } internal ConsistencyLevel AccountConsistencyLevel { get; private set; } internal CosmosResponseFactory ResponseFactory => diff --git a/Microsoft.Azure.Cosmos/src/Handler/ClientPipelineBuilder.cs b/Microsoft.Azure.Cosmos/src/Handler/ClientPipelineBuilder.cs index ff8ee95a87..6b967a24c7 100644 --- a/Microsoft.Azure.Cosmos/src/Handler/ClientPipelineBuilder.cs +++ b/Microsoft.Azure.Cosmos/src/Handler/ClientPipelineBuilder.cs @@ -57,9 +57,9 @@ private set internal CosmosRequestHandler PartitionKeyRangeHandler { get; set; } - public CosmosRequestHandler Build() + public RequestInvokerHandler Build() { - CosmosRequestHandler root = new RequestInvokerHandler(this.client); + RequestInvokerHandler root = new RequestInvokerHandler(this.client); CosmosRequestHandler current = root; if (this.CustomHandlers != null && this.CustomHandlers.Any()) diff --git a/Microsoft.Azure.Cosmos/src/Handler/CosmosRequestMessage.cs b/Microsoft.Azure.Cosmos/src/Handler/CosmosRequestMessage.cs index 5f51ff68e6..6ef7bdb7fc 100644 --- a/Microsoft.Azure.Cosmos/src/Handler/CosmosRequestMessage.cs +++ b/Microsoft.Azure.Cosmos/src/Handler/CosmosRequestMessage.cs @@ -155,8 +155,7 @@ internal async Task AssertPartitioningDetailsAsync(CosmosClient client, Cancella { Debug.Assert(this.AssertPartitioningPropertiesAndHeaders()); } -#endif -#if !DEBUG +#else await Task.CompletedTask; #endif } diff --git a/Microsoft.Azure.Cosmos/src/Handler/RequestInvokerHandler.cs b/Microsoft.Azure.Cosmos/src/Handler/RequestInvokerHandler.cs index bd6cd3dd0b..22e427b3be 100644 --- a/Microsoft.Azure.Cosmos/src/Handler/RequestInvokerHandler.cs +++ b/Microsoft.Azure.Cosmos/src/Handler/RequestInvokerHandler.cs @@ -4,12 +4,15 @@ namespace Microsoft.Azure.Cosmos.Handlers { - using Microsoft.Azure.Cosmos.Internal; - using Microsoft.Azure.Documents; using System; using System.Globalization; + using System.IO; + using System.Net; + using System.Net.Http; using System.Threading; using System.Threading.Tasks; + using Microsoft.Azure.Documents; + using Microsoft.Azure.Documents.Routing; /// /// HttpMessageHandler can only be invoked by derived classed or internal classes inside http assembly @@ -82,6 +85,127 @@ public override Task SendAsync( .Unwrap(); } + public virtual async Task SendAsync( + Uri resourceUri, + ResourceType resourceType, + OperationType operationType, + CosmosRequestOptions requestOptions, + CosmosContainerCore cosmosContainerCore, + Object partitionKey, + Stream streamPayload, + Action requestEnricher, + Func responseCreator, + CancellationToken cancellation = default(CancellationToken)) + { + if (responseCreator == null) + { + throw new ArgumentNullException(nameof(responseCreator)); + } + + CosmosResponseMessage responseMessage = await this.SendAsync( + resourceUri: resourceUri, + resourceType: resourceType, + operationType: operationType, + requestOptions: requestOptions, + cosmosContainerCore: cosmosContainerCore, + partitionKey: partitionKey, + streamPayload: streamPayload, + requestEnricher: requestEnricher, + cancellation: cancellation); + + return responseCreator(responseMessage); + } + + public virtual async Task SendAsync( + Uri resourceUri, + ResourceType resourceType, + OperationType operationType, + CosmosRequestOptions requestOptions, + CosmosContainerCore cosmosContainerCore, + Object partitionKey, + Stream streamPayload, + Action requestEnricher, + CancellationToken cancellation = default(CancellationToken)) + { + if (resourceUri == null) + { + throw new ArgumentNullException(nameof(resourceUri)); + } + + HttpMethod method = RequestInvokerHandler.GetHttpMethod(operationType); + + CosmosRequestMessage request = new CosmosRequestMessage(method, resourceUri); + request.OperationType = operationType; + request.ResourceType = resourceType; + request.RequestOptions = requestOptions; + request.Content = streamPayload; + + if (partitionKey != null) + { + if (cosmosContainerCore == null && Object.ReferenceEquals(partitionKey, CosmosContainerSettings.NonePartitionKeyValue)) + { + throw new ArgumentException($"{nameof(cosmosContainerCore)} can not be null with partition key as PartitionKey.None"); + } + else if (Object.ReferenceEquals(partitionKey, CosmosContainerSettings.NonePartitionKeyValue)) + { + try + { + PartitionKeyInternal partitionKeyInternal = await cosmosContainerCore.GetNonePartitionKeyValueAsync(cancellation); + request.Headers.PartitionKey = partitionKeyInternal.ToJsonString(); + } + catch (DocumentClientException dce) + { + return dce.ToCosmosResponseMessage(request); + } + } + else + { + PartitionKey pk = new PartitionKey(partitionKey); + request.Headers.PartitionKey = pk.InternalKey.ToJsonString(); + } + } + + if (operationType == OperationType.Upsert) + { + request.Headers.IsUpsert = bool.TrueString; + } + + requestEnricher?.Invoke(request); + return await this.SendAsync(request, cancellation); + } + + internal static HttpMethod GetHttpMethod( + OperationType operationType) + { + HttpMethod httpMethod = HttpMethod.Head; + if (operationType == OperationType.Create || + operationType == OperationType.Upsert || + operationType == OperationType.Query || + operationType == OperationType.SqlQuery || + operationType == OperationType.Batch || + operationType == OperationType.ExecuteJavaScript) + { + return HttpMethod.Post; + } + else if (operationType == OperationType.Read || + operationType == OperationType.ReadFeed) + { + return HttpMethod.Get; + } + else if (operationType == OperationType.Replace) + { + return HttpMethod.Put; + } + else if (operationType == OperationType.Delete) + { + return HttpMethod.Delete; + } + else + { + throw new NotImplementedException(); + } + } + private void FillMultiMasterContext(CosmosRequestMessage request) { if (this.client.DocumentClient.UseMultipleWriteLocations) diff --git a/Microsoft.Azure.Cosmos/src/Json/JsonBinaryEncoding.cs b/Microsoft.Azure.Cosmos/src/Json/JsonBinaryEncoding.cs index d6c82f91d1..d9bb24335b 100644 --- a/Microsoft.Azure.Cosmos/src/Json/JsonBinaryEncoding.cs +++ b/Microsoft.Azure.Cosmos/src/Json/JsonBinaryEncoding.cs @@ -85,6 +85,9 @@ internal static class JsonBinaryEncoding "properties", "type", "value", + "Feature", + "FeatureCollection", + "_id", }; /// diff --git a/Microsoft.Azure.Cosmos/src/Microsoft.Azure.Cosmos.csproj b/Microsoft.Azure.Cosmos/src/Microsoft.Azure.Cosmos.csproj index ba41414381..c5c400ca73 100644 --- a/Microsoft.Azure.Cosmos/src/Microsoft.Azure.Cosmos.csproj +++ b/Microsoft.Azure.Cosmos/src/Microsoft.Azure.Cosmos.csproj @@ -6,8 +6,8 @@ This client library enables client applications to connect to Azure Cosmos via the SQL API. Azure Cosmos is a globally distributed, multi-model database service. For more information, refer to http://azure.microsoft.com/services/cosmos-db/. © Microsoft Corporation. All rights reserved. en-US - 3.0.0.10-preview - 3.0.0.26-preview + 3.0.0.11-preview + 3.0.0.28-preview $(ClientVersion)-nightly$(CurrentDate) $(ClientVersion) $(VersionPrefix) @@ -42,11 +42,11 @@ - + - + @@ -61,5 +61,5 @@ $(DefineConstants);DOCDBCLIENT;NETSTANDARD20 $(DefineConstants);SignAssembly - + diff --git a/Microsoft.Azure.Cosmos/src/Query/CosmosQueryContext.cs b/Microsoft.Azure.Cosmos/src/Query/CosmosQueryContext.cs index 787e1e2ee4..afac0b60cf 100644 --- a/Microsoft.Azure.Cosmos/src/Query/CosmosQueryContext.cs +++ b/Microsoft.Azure.Cosmos/src/Query/CosmosQueryContext.cs @@ -36,7 +36,7 @@ internal class CosmosQueryContext public ResourceType ResourceTypeEnum { get; } public OperationType OperationTypeEnum { get; } public Type ResourceType { get; } - public SqlQuerySpec SqlQuerySpec { get; } + public SqlQuerySpec SqlQuerySpec { get; internal set; } public CosmosQueryRequestOptions QueryRequestOptions { get; } public bool IsContinuationExpected { get; } public bool AllowNonValueAggregateQuery { get; } diff --git a/Microsoft.Azure.Cosmos/src/Query/CosmosQueryExecutionContextFactory.cs b/Microsoft.Azure.Cosmos/src/Query/CosmosQueryExecutionContextFactory.cs index e7b22eacae..488bab0894 100644 --- a/Microsoft.Azure.Cosmos/src/Query/CosmosQueryExecutionContextFactory.cs +++ b/Microsoft.Azure.Cosmos/src/Query/CosmosQueryExecutionContextFactory.cs @@ -16,12 +16,14 @@ namespace Microsoft.Azure.Cosmos.Query using Microsoft.Azure.Cosmos.CosmosElements; using Microsoft.Azure.Cosmos.Query.ParallelQuery; using Microsoft.Azure.Documents; + using Microsoft.Azure.Documents.Routing; /// /// Factory class for creating the appropriate DocumentQueryExecutionContext for the provided type of query. /// internal class CosmosQueryExecutionContextFactory : IDocumentQueryExecutionContext { + internal const string InternalPartitionKeyDefinitionProperty = "x-ms-query-partitionkey-definition"; private IDocumentQueryExecutionContext innerExecutionContext; private CosmosQueryContext cosmosQueryContext; @@ -110,11 +112,6 @@ private async Task CreateItemQueryExecutionConte { collection = await collectionCache.ResolveCollectionAsync(request, cancellationToken); } - - if (this.cosmosQueryContext.QueryRequestOptions != null && this.cosmosQueryContext.QueryRequestOptions.PartitionKey != null && this.cosmosQueryContext.QueryRequestOptions.PartitionKey.Equals(PartitionKey.None)) - { - this.cosmosQueryContext.QueryRequestOptions.PartitionKey = PartitionKey.FromInternalKey(collection.GetNoneValue()); - } } if(collection == null) @@ -142,10 +139,31 @@ private async Task CreateItemQueryExecutionConte //todo:elasticcollections this may rely on information from collection cache which is outdated //if collection is deleted/created with same name. //need to make it not rely on information from collection cache. - PartitionedQueryExecutionInfo partitionedQueryExecutionInfo = await GetPartitionedQueryExecutionInfoAsync( - queryClient: this.cosmosQueryContext.QueryClient, + PartitionKeyDefinition partitionKeyDefinition; + object partitionKeyDefinitionObject; + if (this.cosmosQueryContext.QueryRequestOptions?.Properties != null + && this.cosmosQueryContext.QueryRequestOptions.Properties.TryGetValue(InternalPartitionKeyDefinitionProperty, out partitionKeyDefinitionObject)) + { + if (partitionKeyDefinitionObject is PartitionKeyDefinition definition) + { + partitionKeyDefinition = definition; + } + else + { + throw new ArgumentException( + "partitionkeydefinition has invalid type", + nameof(partitionKeyDefinitionObject)); + } + } + else + { + partitionKeyDefinition = collection.PartitionKey; + } + + // $ISSUE-felixfan-2016-07-13: We should probably get PartitionedQueryExecutionInfo from Gateway in GatewayMode + PartitionedQueryExecutionInfo partitionedQueryExecutionInfo = await this.cosmosQueryContext.QueryClient.GetPartitionedQueryExecutionInfoAsync( sqlQuerySpec: this.cosmosQueryContext.SqlQuerySpec, - partitionKeyDefinition: collection.PartitionKey, + partitionKeyDefinition: partitionKeyDefinition, requireFormattableOrderByQuery: true, isContinuationExpected: true, allowNonValueAggregateQuery: this.cosmosQueryContext.AllowNonValueAggregateQuery, @@ -173,6 +191,14 @@ public static async Task CreateSpecializedDocume string collectionRid, CancellationToken cancellationToken) { + if (!string.IsNullOrEmpty(partitionedQueryExecutionInfo.QueryInfo.RewrittenQuery)) + if (!string.IsNullOrEmpty(partitionedQueryExecutionInfo.QueryInfo?.RewrittenQuery)) + { + cosmosQueryContext.SqlQuerySpec = new SqlQuerySpec( + partitionedQueryExecutionInfo.QueryInfo.RewrittenQuery, + cosmosQueryContext.SqlQuerySpec.Parameters); + } + // Figure out the optimal page size. long initialPageSize = cosmosQueryContext.QueryRequestOptions.MaxItemCount.GetValueOrDefault(ParallelQueryConfig.GetConfig().ClientInternalPageSize); @@ -256,10 +282,21 @@ internal static async Task> GetTargetPartitionKeyRanges( List targetRanges; if (queryRequestOptions.PartitionKey != null) { + // Dis-ambiguate the NonePK if used + PartitionKeyInternal partitionKeyInternal = null; + if (Object.ReferenceEquals(queryRequestOptions.PartitionKey, CosmosContainerSettings.NonePartitionKeyValue)) + { + partitionKeyInternal = collection.GetNoneValue(); + } + else + { + partitionKeyInternal = new PartitionKey(queryRequestOptions.PartitionKey).InternalKey; + } + targetRanges = await queryClient.GetTargetPartitionKeyRangesByEpkString( resourceLink, collection.ResourceId, - new PartitionKey(queryRequestOptions.PartitionKey).InternalKey.GetEffectivePartitionKeyString(collection.PartitionKey)); + partitionKeyInternal.GetEffectivePartitionKeyString(collection.PartitionKey)); } else if (TryGetEpkProperty(queryRequestOptions, out string effectivePartitionKeyString)) { @@ -279,7 +316,7 @@ internal static async Task> GetTargetPartitionKeyRanges( return targetRanges; } - public static async Task GetPartitionedQueryExecutionInfoAsync( + public static Task GetPartitionedQueryExecutionInfoAsync( CosmosQueryClient queryClient, SqlQuerySpec sqlQuerySpec, PartitionKeyDefinition partitionKeyDefinition, @@ -290,12 +327,13 @@ public static async Task GetPartitionedQueryExecu { // $ISSUE-felixfan-2016-07-13: We should probably get PartitionedQueryExecutionInfo from Gateway in GatewayMode - QueryPartitionProvider queryPartitionProvider = await queryClient.GetQueryPartitionProviderAsync(cancellationToken); - return queryPartitionProvider.GetPartitionedQueryExecutionInfo(sqlQuerySpec, - partitionKeyDefinition, - requireFormattableOrderByQuery, - isContinuationExpected, - allowNonValueAggregateQuery); + return queryClient.GetPartitionedQueryExecutionInfoAsync( + sqlQuerySpec, + partitionKeyDefinition, + requireFormattableOrderByQuery, + isContinuationExpected, + allowNonValueAggregateQuery, + cancellationToken); } diff --git a/Microsoft.Azure.Cosmos/src/Query/DefaultDocumentQueryExecutionContext.cs b/Microsoft.Azure.Cosmos/src/Query/DefaultDocumentQueryExecutionContext.cs index 069d99fdb9..0eedc91818 100644 --- a/Microsoft.Azure.Cosmos/src/Query/DefaultDocumentQueryExecutionContext.cs +++ b/Microsoft.Azure.Cosmos/src/Query/DefaultDocumentQueryExecutionContext.cs @@ -283,17 +283,56 @@ private static bool ServiceInteropAvailable() { if (this.ShouldExecuteQueryRequest) { + FeedOptions feedOptions = this.GetFeedOptions(null); + PartitionKeyDefinition partitionKeyDefinition; + object partitionKeyDefinitionObject; + if (feedOptions.Properties != null && feedOptions.Properties.TryGetValue(CosmosQueryExecutionContextFactory.InternalPartitionKeyDefinitionProperty, out partitionKeyDefinitionObject)) + { + if (partitionKeyDefinitionObject is PartitionKeyDefinition definition) + { + partitionKeyDefinition = definition; + } + else + { + throw new ArgumentException( + "partitionkeydefinition has invalid type", + nameof(partitionKeyDefinitionObject)); + } + } + else + { + partitionKeyDefinition = collection.PartitionKey; + } + QueryInfo queryInfo; providedRanges = PartitionRoutingHelper.GetProvidedPartitionKeyRanges( this.QuerySpec, enableCrossPartitionQuery, false, isContinuationExpected, - collection.PartitionKey, + partitionKeyDefinition, queryPartitionProvider, version, out queryInfo); } + else if (request.Properties != null && request.Properties.TryGetValue( + WFConstants.BackendHeaders.EffectivePartitionKeyString, + out object effectivePartitionKey)) + { + if (effectivePartitionKey is string effectivePartitionKeyString) + { + providedRanges = new List>() + { + Range.GetPointRange(effectivePartitionKeyString), + }; + } + else + { + throw new ArgumentException( + "EffectivePartitionKey must be a string", + WFConstants.BackendHeaders.EffectivePartitionKeyString); + } + } else { providedRanges = new List> diff --git a/Microsoft.Azure.Cosmos/src/Resource/Container/CosmosContainerCore.cs b/Microsoft.Azure.Cosmos/src/Resource/Container/CosmosContainerCore.cs index 2418469f71..5d63604068 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/Container/CosmosContainerCore.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/Container/CosmosContainerCore.cs @@ -190,7 +190,7 @@ internal Task ReplaceProvisionedThroughputIfExistsAsync( internal async Task GetCachedContainerSettingsAsync(CancellationToken cancellationToken = default(CancellationToken)) { ClientCollectionCache collectionCache = await this.ClientContext.DocumentClient.GetCollectionCacheAsync(); - return await collectionCache.GetByNameAsync(HttpConstants.Versions.CurrentVersion, this.LinkUri.OriginalString, cancellationToken); + return await collectionCache.ResolveByNameAsync(HttpConstants.Versions.CurrentVersion, this.LinkUri.OriginalString, cancellationToken); } // Name based look-up, needs re-computation and can't be cached @@ -213,10 +213,12 @@ internal Task GetRID(CancellationToken cancellationToken) /// The function selects the right partition key constant for inserting documents that don't have /// a value for partition key. The constant selection is based on whether the collection is migrated /// or user partitioned + /// + /// For non-existing container will throw with 404 as status code /// - internal async Task GetNonePartitionKeyValue(CancellationToken cancellationToken = default(CancellationToken)) + internal async Task GetNonePartitionKeyValueAsync(CancellationToken cancellation = default(CancellationToken)) { - CosmosContainerSettings containerSettings = await this.GetCachedContainerSettingsAsync(cancellationToken); + CosmosContainerSettings containerSettings = await this.GetCachedContainerSettingsAsync(cancellation); return containerSettings.GetNoneValue(); } diff --git a/Microsoft.Azure.Cosmos/src/Resource/CosmosClientContext.cs b/Microsoft.Azure.Cosmos/src/Resource/CosmosClientContext.cs index 7442442b6c..e01f17e9d5 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/CosmosClientContext.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/CosmosClientContext.cs @@ -10,6 +10,7 @@ namespace Microsoft.Azure.Cosmos using System.Text; using System.Threading; using System.Threading.Tasks; + using Microsoft.Azure.Cosmos.Handlers; using Microsoft.Azure.Cosmos.Query; using Microsoft.Azure.Documents; @@ -32,7 +33,7 @@ internal abstract class CosmosClientContext internal abstract CosmosResponseFactory ResponseFactory { get; } - internal abstract CosmosRequestHandler RequestHandler { get; } + internal abstract RequestInvokerHandler RequestHandler { get; } internal abstract CosmosClientConfiguration ClientConfiguration { get; } diff --git a/Microsoft.Azure.Cosmos/src/Resource/CosmosClientContextCore.cs b/Microsoft.Azure.Cosmos/src/Resource/CosmosClientContextCore.cs index 97f9c33cbb..287cf49362 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/CosmosClientContextCore.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/CosmosClientContextCore.cs @@ -5,11 +5,11 @@ namespace Microsoft.Azure.Cosmos { using System; - using System.Globalization; using System.IO; using System.Text; using System.Threading; using System.Threading.Tasks; + using Microsoft.Azure.Cosmos.Handlers; using Microsoft.Azure.Cosmos.Query; using Microsoft.Azure.Documents; @@ -20,7 +20,7 @@ internal CosmosClientContextCore( CosmosClientConfiguration clientConfiguration, CosmosJsonSerializer cosmosJsonSerializer, CosmosResponseFactory cosmosResponseFactory, - CosmosRequestHandler requestHandler, + RequestInvokerHandler requestHandler, DocumentClient documentClient, IDocumentQueryClient documentQueryClient) { @@ -46,7 +46,7 @@ internal CosmosClientContextCore( internal override CosmosResponseFactory ResponseFactory { get; } - internal override CosmosRequestHandler RequestHandler { get; } + internal override RequestInvokerHandler RequestHandler { get; } internal override CosmosClientConfiguration ClientConfiguration { get; } @@ -94,17 +94,16 @@ internal override Task ProcessResourceOperationStreamAsyn Action requestEnricher, CancellationToken cancellationToken) { - return ExecUtils.ProcessResourceOperationStreamAsync( - requestHandler: this.RequestHandler, - cosmosContainerCore: cosmosContainerCore, + return this.RequestHandler.SendAsync( resourceUri: resourceUri, resourceType: resourceType, operationType: operationType, requestOptions: requestOptions, + cosmosContainerCore: cosmosContainerCore, partitionKey: partitionKey, streamPayload: streamPayload, requestEnricher: requestEnricher, - cancellationToken: cancellationToken); + cancellation: cancellationToken); } internal override Task ProcessResourceOperationAsync( @@ -117,10 +116,9 @@ internal override Task ProcessResourceOperationAsync( Stream streamPayload, Action requestEnricher, Func responseCreator, - CancellationToken cancellationToken) + CancellationToken cancellation) { - return ExecUtils.ProcessResourceOperationAsync( - requestHandler: this.RequestHandler, + return this.RequestHandler.SendAsync( resourceUri: resourceUri, resourceType: resourceType, operationType: operationType, @@ -130,7 +128,7 @@ internal override Task ProcessResourceOperationAsync( streamPayload: streamPayload, requestEnricher: requestEnricher, responseCreator: responseCreator, - cancellationToken: cancellationToken); + cancellation: cancellation); } } } diff --git a/Microsoft.Azure.Cosmos/src/Resource/Query/CosmosQueryClient.cs b/Microsoft.Azure.Cosmos/src/Resource/Query/CosmosQueryClient.cs index df6112043d..14f0bf5044 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/Query/CosmosQueryClient.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/Query/CosmosQueryClient.cs @@ -23,7 +23,13 @@ internal abstract class CosmosQueryClient internal abstract Task GetRoutingMapProviderAsync(); - internal abstract Task GetQueryPartitionProviderAsync(CancellationToken cancellationToken); + internal abstract Task GetPartitionedQueryExecutionInfoAsync( + SqlQuerySpec sqlQuerySpec, + PartitionKeyDefinition partitionKeyDefinition, + bool requireFormattableOrderByQuery, + bool isContinuationExpected, + bool allowNonValueAggregateQuery, + CancellationToken cancellationToken); internal abstract Task> ExecuteItemQueryAsync( Uri resourceUri, diff --git a/Microsoft.Azure.Cosmos/src/Resource/Query/CosmosQueryClientCore.cs b/Microsoft.Azure.Cosmos/src/Resource/Query/CosmosQueryClientCore.cs index 068896faeb..6412741bcf 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/Query/CosmosQueryClientCore.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/Query/CosmosQueryClientCore.cs @@ -49,9 +49,21 @@ internal override Task GetRoutingMapProviderAsync() return this.DocumentQueryClient.GetRoutingMapProviderAsync(); } - internal override Task GetQueryPartitionProviderAsync(CancellationToken cancellationToken) + internal override async Task GetPartitionedQueryExecutionInfoAsync( + SqlQuerySpec sqlQuerySpec, + PartitionKeyDefinition partitionKeyDefinition, + bool requireFormattableOrderByQuery, + bool isContinuationExpected, + bool allowNonValueAggregateQuery, + CancellationToken cancellationToken) { - return this.DocumentQueryClient.GetQueryPartitionProviderAsync(cancellationToken); + QueryPartitionProvider queryPartitionProvider = await this.DocumentQueryClient.GetQueryPartitionProviderAsync(cancellationToken); + return queryPartitionProvider.GetPartitionedQueryExecutionInfo( + sqlQuerySpec, + partitionKeyDefinition, + requireFormattableOrderByQuery, + isContinuationExpected, + allowNonValueAggregateQuery); } internal override async Task> ExecuteItemQueryAsync( diff --git a/Microsoft.Azure.Cosmos/src/Resource/QueryResponses/CosmosResultSetIteratorCore.cs b/Microsoft.Azure.Cosmos/src/Resource/QueryResponses/CosmosResultSetIteratorCore.cs index d285ce3b84..e6716ff8b8 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/QueryResponses/CosmosResultSetIteratorCore.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/QueryResponses/CosmosResultSetIteratorCore.cs @@ -66,18 +66,14 @@ internal CosmosResultSetIteratorCore( /// /// (Optional) representing request cancellation. /// A query response from cosmos service - public override Task FetchNextSetAsync(CancellationToken cancellationToken = default(CancellationToken)) + public override async Task FetchNextSetAsync(CancellationToken cancellationToken = default(CancellationToken)) { cancellationToken.ThrowIfCancellationRequested(); - return this.nextResultSetDelegate(this.continuationToken, this.queryOptions, this.state, cancellationToken) - .ContinueWith(task => - { - CosmosQueryResponse response = task.Result; - this.continuationToken = response.ContinuationToken; - this.HasMoreResults = response.GetHasMoreResults(); - return response; - }, cancellationToken); + CosmosQueryResponse response = await this.nextResultSetDelegate(this.continuationToken, this.queryOptions, this.state, cancellationToken); + this.continuationToken = response.ContinuationToken; + this.HasMoreResults = response.GetHasMoreResults(); + return response; } } @@ -141,19 +137,15 @@ internal CosmosDefaultResultSetIterator( /// /// (Optional) representing request cancellation. /// A query response from cosmos service - public override Task> FetchNextSetAsync(CancellationToken cancellationToken = default(CancellationToken)) + public override async Task> FetchNextSetAsync(CancellationToken cancellationToken = default(CancellationToken)) { cancellationToken.ThrowIfCancellationRequested(); - return this.nextResultSetDelegate(this.MaxItemCount, this.continuationToken, this.queryOptions, this.state, cancellationToken) - .ContinueWith(task => - { - CosmosQueryResponse response = task.Result; - this.HasMoreResults = response.GetHasMoreResults(); - this.continuationToken = response.InternalContinuationToken; - - return response; - }, cancellationToken); + CosmosQueryResponse response = await this.nextResultSetDelegate(this.MaxItemCount, this.continuationToken, this.queryOptions, this.state, cancellationToken); + this.HasMoreResults = response.GetHasMoreResults(); + this.continuationToken = response.InternalContinuationToken; + return response; + } internal static CosmosQueryResponse CreateCosmosQueryResponse( diff --git a/Microsoft.Azure.Cosmos/src/Routing/ClientCollectionCache.cs b/Microsoft.Azure.Cosmos/src/Routing/ClientCollectionCache.cs index b0c3799efd..212365d69e 100644 --- a/Microsoft.Azure.Cosmos/src/Routing/ClientCollectionCache.cs +++ b/Microsoft.Azure.Cosmos/src/Routing/ClientCollectionCache.cs @@ -45,7 +45,7 @@ protected override Task GetByRidAsync(string apiVersion cancellationToken); } - internal override Task GetByNameAsync(string apiVersion, string resourceAddress, CancellationToken cancellationToken) + protected override Task GetByNameAsync(string apiVersion, string resourceAddress, CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); IDocumentClientRetryPolicy retryPolicyInstance = new ClearingSessionContainerClientRetryPolicy(this.sessionContainer, this.retryPolicy.GetRequestPolicy()); diff --git a/Microsoft.Azure.Cosmos/src/Routing/CollectionCache.cs b/Microsoft.Azure.Cosmos/src/Routing/CollectionCache.cs index f39196d833..5653953f22 100644 --- a/Microsoft.Azure.Cosmos/src/Routing/CollectionCache.cs +++ b/Microsoft.Azure.Cosmos/src/Routing/CollectionCache.cs @@ -187,7 +187,7 @@ public void Refresh(string resourceAddress, string apiVersion = null) protected abstract Task GetByRidAsync(string apiVersion, string collectionRid, CancellationToken cancellationToken); - internal abstract Task GetByNameAsync(string apiVersion, string resourceAddress, CancellationToken cancellationToken); + protected abstract Task GetByNameAsync(string apiVersion, string resourceAddress, CancellationToken cancellationToken); private async Task ResolveByPartitionKeyRangeIdentityAsync(string apiVersion, PartitionKeyRangeIdentity partitionKeyRangeIdentity, CancellationToken cancellationToken) { @@ -234,7 +234,7 @@ private Task ResolveByRidAsync( cancellationToken); } - private async Task ResolveByNameAsync( + internal virtual async Task ResolveByNameAsync( string apiVersion, string resourceAddress, CancellationToken cancellationToken) diff --git a/Microsoft.Azure.Cosmos/src/Util/ExecUtils.cs b/Microsoft.Azure.Cosmos/src/Util/ExecUtils.cs index f115d6003c..fcbaebbdd1 100644 --- a/Microsoft.Azure.Cosmos/src/Util/ExecUtils.cs +++ b/Microsoft.Azure.Cosmos/src/Util/ExecUtils.cs @@ -6,10 +6,9 @@ namespace Microsoft.Azure.Cosmos { using System; using System.IO; - using System.Net.Http; using System.Threading; using System.Threading.Tasks; - using Microsoft.Azure.Documents.Routing; + using Microsoft.Azure.Cosmos.Handlers; using Microsoft.Azure.Documents; internal static class ExecUtils @@ -122,7 +121,7 @@ internal static Task ProcessResourceOperationAsync( } internal static async Task ProcessResourceOperationAsync( - CosmosRequestHandler requestHandler, + RequestInvokerHandler requestHandler, Uri resourceUri, ResourceType resourceType, OperationType operationType, @@ -149,7 +148,7 @@ internal static async Task ProcessResourceOperationAsync( throw new ArgumentNullException(nameof(responseCreator)); } - CosmosRequestMessage request = await ExecUtils.GenerateCosmosRequestMessage( + CosmosResponseMessage response = await requestHandler.SendAsync( resourceUri, resourceType, operationType, @@ -159,141 +158,7 @@ internal static async Task ProcessResourceOperationAsync( streamPayload, requestEnricher); - CosmosResponseMessage response = await requestHandler.SendAsync(request, cancellationToken); return responseCreator(response); } - - /// - /// Used internally by friends ensure robust argument and - /// exception-less handling - /// - internal static Task ProcessResourceOperationAsync( - CosmosClient client, - Uri resourceUri, - ResourceType resourceType, - OperationType operationType, - CosmosRequestOptions requestOptions, - Object partitionKey, - Stream streamPayload, - Action requestEnricher, - Func responseCreator, - CancellationToken cancellationToken) - { - return ProcessResourceOperationAsync( - requestHandler: client.RequestHandler, - resourceUri: resourceUri, - resourceType: resourceType, - operationType: operationType, - requestOptions: requestOptions, - cosmosContainerCore: null, - partitionKey: partitionKey, - streamPayload: streamPayload, - requestEnricher: requestEnricher, - responseCreator: responseCreator, - cancellationToken: cancellationToken); - } - - internal static async Task ProcessResourceOperationStreamAsync( - CosmosRequestHandler requestHandler, - Uri resourceUri, - ResourceType resourceType, - OperationType operationType, - CosmosRequestOptions requestOptions, - CosmosContainerCore cosmosContainerCore, - Object partitionKey, - Stream streamPayload, - Action requestEnricher, - CancellationToken cancellationToken) - { - CosmosRequestMessage request = await ExecUtils.GenerateCosmosRequestMessage( - resourceUri, - resourceType, - operationType, - requestOptions, - cosmosContainerCore, - partitionKey, - streamPayload, - requestEnricher); - - return await requestHandler.SendAsync(request, cancellationToken); - } - - private static async Task GenerateCosmosRequestMessage( - Uri resourceUri, - ResourceType resourceType, - OperationType operationType, - CosmosRequestOptions requestOptions, - CosmosContainerCore cosmosContainerCore, - Object partitionKey, - Stream streamPayload, - Action requestEnricher) - { - HttpMethod method = ExecUtils.GetHttpMethod(operationType); - - CosmosRequestMessage request = new CosmosRequestMessage(method, resourceUri); - request.OperationType = operationType; - request.ResourceType = resourceType; - request.RequestOptions = requestOptions; - request.Content = streamPayload; - - if (partitionKey != null) - { - if (cosmosContainerCore == null && partitionKey.Equals(PartitionKey.None)) - { - throw new ArgumentException($"{nameof(cosmosContainerCore)} can not be null with partition key as PartitionKey.None"); - } - else if (partitionKey.Equals(PartitionKey.None)) - { - PartitionKeyInternal partitionKeyInternal = await cosmosContainerCore.GetNonePartitionKeyValue(); - request.Headers.PartitionKey = partitionKeyInternal.ToJsonString(); - } - else - { - PartitionKey pk = new PartitionKey(partitionKey); - request.Headers.PartitionKey = pk.InternalKey.ToJsonString(); - } - } - - if (operationType == OperationType.Upsert) - { - request.Headers.IsUpsert = bool.TrueString; - } - - requestEnricher?.Invoke(request); - - return request; - } - - internal static HttpMethod GetHttpMethod( - OperationType operationType) - { - HttpMethod httpMethod = HttpMethod.Head; - if (operationType == OperationType.Create || - operationType == OperationType.Upsert || - operationType == OperationType.Query || - operationType == OperationType.SqlQuery || - operationType == OperationType.Batch || - operationType == OperationType.ExecuteJavaScript) - { - return HttpMethod.Post; - } - else if (operationType == OperationType.Read || - operationType == OperationType.ReadFeed) - { - return HttpMethod.Get; - } - else if (operationType == OperationType.Replace) - { - return HttpMethod.Put; - } - else if (operationType == OperationType.Delete) - { - return HttpMethod.Delete; - } - else - { - throw new NotImplementedException(); - } - } } } \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosContainerTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosContainerTests.cs index fddbaa1eaf..f090a92c29 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosContainerTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosContainerTests.cs @@ -64,6 +64,7 @@ public async Task ContainerContractTest() Assert.IsTrue(containerSettings.LastModified.Value > new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc), containerSettings.LastModified.Value.ToString()); } + [Ignore] [TestMethod] public async Task PartitionedCRUDTest() { diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosItemTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosItemTests.cs index 29642ae636..5b652b6ae6 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosItemTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosItemTests.cs @@ -14,6 +14,7 @@ namespace Microsoft.Azure.Cosmos.SDK.EmulatorTests using System.Net; using System.Net.Http; using System.Text; + using System.Threading; using System.Threading.Tasks; using Microsoft.Azure.Cosmos.Json; using Microsoft.Azure.Cosmos.Query; @@ -22,6 +23,7 @@ namespace Microsoft.Azure.Cosmos.SDK.EmulatorTests using Microsoft.Azure.Documents; using Microsoft.VisualStudio.TestTools.UnitTesting; using Newtonsoft.Json; + using static Microsoft.Azure.Cosmos.SDK.EmulatorTests.TransportWrapperTests; using JsonReader = Json.JsonReader; using JsonWriter = Json.JsonWriter; @@ -91,6 +93,109 @@ public async Task CreateDropItemUndefinedPartitionKeyTest() Assert.IsNotNull(deleteResponse); } + [TestMethod] + public async Task ReadCollectionNotExists() + { + string collectionName = Guid.NewGuid().ToString(); + CosmosContainer testContainer = this.database.Containers[collectionName]; + await CosmosItemTests.TestNonePKForNonExistingContainer(testContainer); + + // Item -> Container -> Database contract + string dbName = Guid.NewGuid().ToString(); + testContainer = this.cosmosClient.Databases[dbName].Containers[collectionName]; + await CosmosItemTests.TestNonePKForNonExistingContainer(testContainer); + } + + [TestMethod] + public async Task NonPartitionKeyLookupCacheTest() + { + int count = 0; + CosmosClient client = TestCommon.CreateCosmosClient(builder => + { + builder.UseConnectionModeDirect(); + builder.UseSendingRequestEventArgs((sender, e) => + { + if (e.DocumentServiceRequest != null) + { + Trace.TraceInformation($"{e.DocumentServiceRequest.ToString()}"); + } + + if (e.HttpRequest != null) + { + Trace.TraceInformation($"{e.HttpRequest.ToString()}"); + } + + if (e.IsHttpRequest() + && e.HttpRequest.RequestUri.AbsolutePath.Contains("/colls/")) + { + count++; + } + + if (e.IsHttpRequest() + && e.HttpRequest.RequestUri.AbsolutePath.Contains("/pkranges")) + { + Debugger.Break(); + } + }); + }); + + string dbName = Guid.NewGuid().ToString(); + string containerName = Guid.NewGuid().ToString(); + CosmosContainerCore testContainer = (CosmosContainerCore)client.Databases[dbName].Containers[containerName]; + + int loopCount = 2; + for (int i = 0; i < loopCount; i++) + { + try + { + await testContainer.GetNonePartitionKeyValueAsync(); + Assert.Fail(); + } + catch (DocumentClientException dce) when (dce.StatusCode == HttpStatusCode.NotFound) + { + } + } + + Assert.AreEqual(loopCount, count); + + // Create real container and address + CosmosDatabase db = await client.Databases.CreateDatabaseAsync(dbName); + CosmosContainer container = await db.Containers.CreateContainerAsync(containerName, "/id"); + + // reset counter + count = 0; + for (int i = 0; i < loopCount; i++) + { + await testContainer.GetNonePartitionKeyValueAsync(); + } + + // expected once post create + Assert.AreEqual(1, count); + + // reset counter + count = 0; + for (int i = 0; i < loopCount; i++) + { + await testContainer.GetRID(default(CancellationToken)); + } + + // Already cached by GetNonePartitionKeyValueAsync before + Assert.AreEqual(0, count); + + // reset counter + count = 0; + int expected = 0; + for (int i = 0; i < loopCount; i++) + { + await testContainer.GetRoutingMapAsync(default(CancellationToken)); + expected = count; + } + + // OkRagnes should be fetched only once. + // Possible to make multiple calls for ranges + Assert.AreEqual(expected, count); + } + [TestMethod] public async Task CreateDropItemStreamTest() { @@ -635,20 +740,8 @@ public async Task NegativeQueryTest() await resultSet.FetchNextSetAsync(); Assert.Fail("Expected query to fail"); } - catch (AggregateException e) + catch (CosmosException exception) when (exception.StatusCode == HttpStatusCode.BadRequest) { - CosmosException exception = e.InnerException as CosmosException; - - if (exception == null) - { - throw e; - } - - if (exception.StatusCode != HttpStatusCode.BadRequest) - { - throw e; - } - Assert.IsTrue(exception.Message.Contains("continuation token limit specified is not large enough")); } @@ -661,20 +754,8 @@ public async Task NegativeQueryTest() await resultSet.FetchNextSetAsync(); Assert.Fail("Expected query to fail"); } - catch (AggregateException e) + catch (CosmosException exception) when (exception.StatusCode == HttpStatusCode.BadRequest) { - CosmosException exception = e.InnerException as CosmosException; - - if (exception == null) - { - throw e; - } - - if (exception.StatusCode != HttpStatusCode.BadRequest) - { - throw e; - } - Assert.IsTrue(exception.Message.Contains("Syntax error, incorrect syntax near")); } } @@ -723,7 +804,6 @@ public async Task ItemRequestOptionAccessConditionTest() // Read write non partition Container item. [TestMethod] - [Ignore] //Temporary ignore till we fix emulator issue public async Task ReadNonPartitionItemAsync() { try @@ -873,7 +953,18 @@ private async Task> CreateRandomItems(int pkCount, int perPK return createdList; } - private async Task CreateNonPartitionContainerItem() + private Task CreateNonPartitionContainerItem() + { + string itemDefinition = JsonConvert.SerializeObject(this.CreateRandomToDoActivity(id: nonPartitionItemId)); + return CosmosItemTests.CreateNonPartitionContainerItem(this.database.Id, + CosmosItemTests.nonPartitionContainerId, + itemDefinition); + } + + internal static async Task CreateNonPartitionContainerItem( + string dbName, + string containerName, + string itemDefinition = null) { string authKey = ConfigurationManager.AppSettings["MasterKey"]; string endpoint = ConfigurationManager.AppSettings["GatewayEndpoint"]; @@ -882,33 +973,37 @@ private async Task CreateNonPartitionContainerItem() Uri baseUri = new Uri(endpoint); string verb = "POST"; string resourceType = "colls"; - string resourceId = string.Format("dbs/{0}", this.database.Id); - string resourceLink = string.Format("dbs/{0}/colls", this.database.Id); + string resourceId = string.Format("dbs/{0}", dbName); + string resourceLink = string.Format("dbs/{0}/colls", dbName); client.DefaultRequestHeaders.Add("x-ms-date", utc_date); client.DefaultRequestHeaders.Add("x-ms-version", "2018-09-17"); - string authHeader = this.GenerateMasterKeyAuthorizationSignature(verb, resourceId, resourceType, authKey, "master", "1.0"); + string authHeader = CosmosItemTests.GenerateMasterKeyAuthorizationSignature(verb, resourceId, resourceType, authKey, "master", "1.0"); client.DefaultRequestHeaders.Add("authorization", authHeader); - string containerDefinition = "{\n \"id\": \"" + nonPartitionContainerId + "\"\n}"; + string containerDefinition = "{\n \"id\": \"" + containerName + "\"\n}"; StringContent containerContent = new StringContent(containerDefinition); Uri requestUri = new Uri(baseUri, resourceLink); - await client.PostAsync(requestUri.ToString(), containerContent); + HttpResponseMessage response = await client.PostAsync(requestUri.ToString(), containerContent); + Assert.IsTrue(response.StatusCode == HttpStatusCode.Created || response.StatusCode == HttpStatusCode.Conflict, response.ToString()); //Creating non partition Container item. verb = "POST"; resourceType = "docs"; - resourceId = string.Format("dbs/{0}/colls/{1}", this.database.Id, nonPartitionContainerId); - resourceLink = string.Format("dbs/{0}/colls/{1}/docs", this.database.Id, nonPartitionContainerId); - authHeader = this.GenerateMasterKeyAuthorizationSignature(verb, resourceId, resourceType, authKey, "master", "1.0"); + resourceId = string.Format("dbs/{0}/colls/{1}", dbName, containerName); + resourceLink = string.Format("dbs/{0}/colls/{1}/docs", dbName, containerName); + authHeader = CosmosItemTests.GenerateMasterKeyAuthorizationSignature(verb, resourceId, resourceType, authKey, "master", "1.0"); client.DefaultRequestHeaders.Remove("authorization"); client.DefaultRequestHeaders.Add("authorization", authHeader); - string itemDefinition = JsonConvert.SerializeObject(this.CreateRandomToDoActivity(id: nonPartitionItemId)); - StringContent itemContent = new StringContent(itemDefinition); - requestUri = new Uri(baseUri, resourceLink); - await client.PostAsync(requestUri.ToString(), itemContent); + if (!string.IsNullOrEmpty(itemDefinition)) + { + StringContent itemContent = new StringContent(itemDefinition); + requestUri = new Uri(baseUri, resourceLink); + response = await client.PostAsync(requestUri.ToString(), itemContent); + Assert.IsTrue(response.StatusCode == HttpStatusCode.Created || response.StatusCode == HttpStatusCode.Conflict, response.ToString()); + } } private async Task CreateUndefinedPartitionItem() @@ -931,7 +1026,7 @@ private async Task CreateUndefinedPartitionItem() resourceType = "docs"; resourceId = string.Format("dbs/{0}/colls/{1}", this.database.Id, this.Container.Id); resourceLink = string.Format("dbs/{0}/colls/{1}/docs", this.database.Id, this.Container.Id); - string authHeader = this.GenerateMasterKeyAuthorizationSignature(verb, resourceId, resourceType, authKey, "master", "1.0"); + string authHeader = CosmosItemTests.GenerateMasterKeyAuthorizationSignature(verb, resourceId, resourceType, authKey, "master", "1.0"); client.DefaultRequestHeaders.Remove("authorization"); client.DefaultRequestHeaders.Add("authorization", authHeader); @@ -943,7 +1038,7 @@ private async Task CreateUndefinedPartitionItem() await client.PostAsync(requestUri.ToString(), itemContent); } - private string GenerateMasterKeyAuthorizationSignature(string verb, string resourceId, string resourceType, string key, string keyType, string tokenVersion) + private static string GenerateMasterKeyAuthorizationSignature(string verb, string resourceId, string resourceType, string key, string keyType, string tokenVersion) { System.Security.Cryptography.HMACSHA256 hmacSha256 = new System.Security.Cryptography.HMACSHA256 { Key = Convert.FromBase64String(key) }; @@ -1024,5 +1119,19 @@ private ToDoActivityAfterMigration CreateRandomToDoActivityAfterMigration(string cost = double.MaxValue }; } + + private static async Task TestNonePKForNonExistingContainer(CosmosContainer cosmosContainer) + { + // Stream implementation should not throw + CosmosResponseMessage response = await cosmosContainer.Items.ReadItemStreamAsync(CosmosContainerSettings.NonePartitionKeyValue, "id1"); + Assert.AreEqual(HttpStatusCode.NotFound, response.StatusCode); + Assert.IsNotNull(response.Headers.ActivityId); + Assert.IsNotNull(response.ErrorMessage); + + // FOr typed also its not error + var typedResponse = await cosmosContainer.Items.ReadItemAsync(CosmosContainerSettings.NonePartitionKeyValue, "id1"); + Assert.AreEqual(HttpStatusCode.NotFound, typedResponse.StatusCode); + Assert.IsNotNull(typedResponse.Headers.ActivityId); + } } } diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CrossPartitionQueryTests.AggregateMixedTypes_baseline.xml b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CrossPartitionQueryTests.AggregateMixedTypes_baseline.xml index 450e8cc453..6ec5126974 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CrossPartitionQueryTests.AggregateMixedTypes_baseline.xml +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CrossPartitionQueryTests.AggregateMixedTypes_baseline.xml @@ -189,13 +189,13 @@ WHERE c.key = "arrayOnlyKey"]]> - + - + - + - + +// Copyright (c) Microsoft Corporation. All rights reserved. +// +//----------------------------------------------------------------------- +namespace Microsoft.Azure.Cosmos.SDK.EmulatorTests +{ + using System; + using System.Collections.Generic; + using System.Collections.ObjectModel; + using System.Diagnostics; + using System.Globalization; + using System.IO; + using System.Linq; + using System.Net; + using System.Text; + using System.Text.RegularExpressions; + using System.Threading; + using System.Threading.Tasks; + using System.Xml; + using Microsoft.Azure.Cosmos.CosmosElements; + using Microsoft.Azure.Cosmos.Routing; + using Microsoft.Azure.Documents; + using Microsoft.Azure.Documents.Routing; + using Microsoft.VisualStudio.TestTools.UnitTesting; + using Newtonsoft.Json; + using Newtonsoft.Json.Converters; + using Newtonsoft.Json.Linq; + using Query; + using Query.ParallelQuery; + + /// + /// Tests for CrossPartitionQueryTestsOnePartition. + /// + [TestCategory("Quarantine")] + [TestClass] + public class CrossPartitionQueryTestsOnePartition + { + private static readonly string[] NoDocuments = new string[] { }; + private CosmosClient GatewayClient = TestCommon.CreateCosmosClient(true); + private CosmosClient Client = TestCommon.CreateCosmosClient(false); + private CosmosDatabase database; + private readonly AsyncLocal responseLengthBytes = new AsyncLocal(); + private readonly AsyncLocal outerCosmosQueryResponseActivityId = new AsyncLocal(); + + [FlagsAttribute] + private enum ConnectionModes + { + None = 0, + Direct = 0x1, + Gateway = 0x2, + } + + [TestInitialize] + public async Task Initialize() + { + await this.CleanUp(); + this.database = await this.Client.Databases.CreateDatabaseAsync(Guid.NewGuid().ToString() + "db"); + } + + [TestCleanup] + public async Task Cleanup() + { + await this.database.DeleteAsync(); + } + + private static string GetApiVersion() + { + return HttpConstants.Versions.CurrentVersion; + } + + private static void SetApiVersion(string apiVersion) + { + HttpConstants.Versions.CurrentVersion = apiVersion; + HttpConstants.Versions.CurrentVersionUTF8 = Encoding.UTF8.GetBytes(apiVersion); + } + + private async Task> GetPartitionKeyRanges(CosmosContainerSettings container) + { + Range fullRange = new Range( + PartitionKeyInternal.MinimumInclusiveEffectivePartitionKey, + PartitionKeyInternal.MaximumExclusiveEffectivePartitionKey, + true, + false); + IRoutingMapProvider routingMapProvider = await this.Client.DocumentClient.GetPartitionKeyRangeCacheAsync(); + IReadOnlyList ranges = await routingMapProvider.TryGetOverlappingRangesAsync(container.ResourceId, fullRange); + return ranges; + } + + private async Task CreatePartitionContainer(string partitionKey = "/id", Microsoft.Azure.Cosmos.IndexingPolicy indexingPolicy = null) + { + CosmosContainerResponse containerResponse = await this.database.Containers.CreateContainerAsync( + new CosmosContainerSettings + { + Id = Guid.NewGuid().ToString() + "container", + IndexingPolicy = indexingPolicy == null ? new Cosmos.IndexingPolicy + { + IncludedPaths = new Collection + { + new Cosmos.IncludedPath + { + Path = "/*", + Indexes = new Collection + { + Cosmos.Index.Range(Cosmos.DataType.Number), + Cosmos.Index.Range(Cosmos.DataType.String), + } + } + } + } : indexingPolicy, + PartitionKey = new PartitionKeyDefinition + { + Paths = new Collection { partitionKey }, + Kind = PartitionKind.Hash + } + }, + // This throughput needs to be about half the max with multi master + // otherwise it will create about twice as many partitions. + 5000); + + IReadOnlyList ranges = await this.GetPartitionKeyRanges(containerResponse); + Assert.AreEqual(1, ranges.Count()); + + return containerResponse; + } + + private async Task>> CreatePartitionedContainerAndIngestDocuments(IEnumerable documents, string partitionKey = "/id", Cosmos.IndexingPolicy indexingPolicy = null) + { + CosmosContainer partitionedCollection = await this.CreatePartitionContainer(partitionKey, indexingPolicy); + List insertedDocuments = new List(); + string jObjectPartitionKey = partitionKey.Remove(0, 1); + foreach (string document in documents) + { + JObject documentObject = JsonConvert.DeserializeObject(document); + if (documentObject["id"] == null) + { + documentObject["id"] = Guid.NewGuid().ToString(); + } + + JValue pkToken = (JValue)documentObject[jObjectPartitionKey]; + object pkValue = pkToken != null ? pkToken.Value : Undefined.Value; + insertedDocuments.Add((await partitionedCollection.Items.CreateItemAsync(pkValue, documentObject)).Resource.ToObject()); + + } + + return new Tuple>(partitionedCollection, insertedDocuments); + } + + private async Task CleanUp() + { + CosmosResultSetIterator allDatabases = this.Client.Databases.GetDatabaseIterator(); + + while (allDatabases.HasMoreResults) + { + foreach (CosmosDatabaseSettings db in await allDatabases.FetchNextSetAsync()) + { + await this.Client.Databases[db.Id].DeleteAsync(); + } + } + } + + private async Task RunWithApiVersion(string apiVersion, Func function) + { + string originalApiVersion = GetApiVersion(); + CosmosClient originalCosmosClient = this.Client; + CosmosClient originalGatewayClient = this.GatewayClient; + CosmosDatabase originalDatabase = this.database; + + try + { + SetApiVersion(apiVersion); + if (apiVersion != originalApiVersion) + { + this.Client = TestCommon.CreateCosmosClient(false); + this.GatewayClient = TestCommon.CreateCosmosClient(true); + this.database = this.Client.Databases[this.database.Id]; + } + + await function(); + } + finally + { + this.Client = originalCosmosClient; + this.GatewayClient = originalGatewayClient; + this.database = originalDatabase; + SetApiVersion(originalApiVersion); + } + } + + internal delegate Task Query( + CosmosContainer container, + IEnumerable documents); + + internal delegate Task Query( + CosmosContainer container, + IEnumerable documents, + T testArgs); + + internal delegate CosmosClient CosmosClientFactory(ConnectionMode connectionMode); + + private async Task CreateIngestQueryDelete( + ConnectionModes connectionModes, + IEnumerable documents, + Query query, + string partitionKey = "/id", + Cosmos.IndexingPolicy indexingPolicy = null, + CosmosClientFactory cosmosClientFactory = null) + { + Query queryWrapper = (container, inputDocuments, throwaway) => + { + return query(container, inputDocuments); + }; + + await this.CreateIngestQueryDelete( + connectionModes, + documents, + queryWrapper, + null, + partitionKey, + indexingPolicy, + cosmosClientFactory); + } + + private async Task CreateIngestQueryDelete( + ConnectionModes connectionModes, + IEnumerable documents, + Query query, + T testArgs, + string partitionKey = "/id", + Cosmos.IndexingPolicy indexingPolicy = null, + CosmosClientFactory cosmosClientFactory = null) + { + await this.CreateIngestQueryDelete( + connectionModes, + documents, + query, + cosmosClientFactory ?? this.CreateDefaultCosmosClient, + testArgs, + partitionKey, + indexingPolicy); + } + + /// + /// Task that wraps boiler plate code for query tests (container create -> ingest documents -> query documents -> delete collections). + /// Note that this function will take the cross product connectionModes + /// + /// The connection modes to use. + /// The documents to ingest + /// + /// The callback for the queries. + /// All the standard arguments will be passed in. + /// Please make sure that this function is idempotent, since a container will be reused for each connection mode. + /// + /// + /// The callback for the create CosmosClient. This is invoked for the different ConnectionModes that the query is targeting. + /// If CosmosClient instantiated by this does not apply the expected ConnectionMode, an assert is thrown. + /// + /// The partition key for the partition container. + /// The optional args that you want passed in to the query. + /// A task to await on. + private async Task CreateIngestQueryDelete( + ConnectionModes connectionModes, + IEnumerable documents, + Query query, + CosmosClientFactory cosmosClientFactory, + T testArgs, + string partitionKey = "/id", + Cosmos.IndexingPolicy indexingPolicy = null) + { + int retryCount = 1; + AggregateException exceptionHistory = new AggregateException(); + while (retryCount-- > 0) + { + try + { + List>>> createContainerTasks = new List>>> + { + this.CreatePartitionedContainerAndIngestDocuments(documents, partitionKey, indexingPolicy) + }; + + Tuple>[] collectionsAndDocuments = await Task.WhenAll(createContainerTasks); + + List cosmosClients = new List(); + foreach (ConnectionModes connectionMode in Enum.GetValues(connectionModes.GetType()).Cast().Where(connectionModes.HasFlag)) + { + if (connectionMode == ConnectionModes.None) + { + continue; + } + + ConnectionMode targetConnectionMode = GetTargetConnectionMode(connectionMode); + CosmosClient cosmosClient = cosmosClientFactory(targetConnectionMode); + + Assert.AreEqual(targetConnectionMode, cosmosClient.Configuration.ConnectionMode, "Test setup: Invalid connection policy applied to CosmosClient"); + cosmosClients.Add(cosmosClient); + } + + bool succeeded = false; + while (!succeeded) + { + try + { + List queryTasks = new List(); + foreach (CosmosClient cosmosClient in cosmosClients) + { + foreach (Tuple> containerAndDocuments in collectionsAndDocuments) + { + CosmosContainer container = cosmosClient.Databases[containerAndDocuments.Item1.Database.Id].Containers[containerAndDocuments.Item1.Id]; + queryTasks.Add(query(container, containerAndDocuments.Item2, testArgs)); + } + } + + await Task.WhenAll(queryTasks); + succeeded = true; + } + catch (TaskCanceledException) + { + // SDK throws TaskCanceledException every now and then + } + } + + List> deleteContainerTasks = new List>(); + foreach (CosmosContainer container in collectionsAndDocuments.Select(tuple => tuple.Item1)) + { + deleteContainerTasks.Add(container.DeleteAsync()); + } + + await Task.WhenAll(deleteContainerTasks); + + // If you made it here then it's all good + break; + } + catch (Exception ex) + { + if (ex.GetType() == typeof(AssertFailedException)) + { + throw; + } + else + { + List previousExceptions = exceptionHistory.InnerExceptions.ToList(); + previousExceptions.Add(ex); + exceptionHistory = new AggregateException(previousExceptions); + } + } + } + + if (exceptionHistory.InnerExceptions.Count > 0) + { + throw exceptionHistory; + } + } + + private static ConnectionMode GetTargetConnectionMode(ConnectionModes connectionMode) + { + ConnectionMode targetConnectionMode = ConnectionMode.Gateway; + switch (connectionMode) + { + case ConnectionModes.Gateway: + targetConnectionMode = ConnectionMode.Gateway; + break; + + case ConnectionModes.Direct: + targetConnectionMode = ConnectionMode.Direct; + break; + + default: + throw new ArgumentException($"Unexpected connection mode: {connectionMode}"); + } + + return targetConnectionMode; + } + + private CosmosClient CreateDefaultCosmosClient(ConnectionMode connectionMode) + { + switch (connectionMode) + { + case ConnectionMode.Gateway: + return this.GatewayClient; + case ConnectionMode.Direct: + return this.Client; + default: + throw new ArgumentException($"Unexpected connection mode: {connectionMode}"); + } + } + + private CosmosClient CreateNewCosmosClient(ConnectionMode connectionMode) + { + switch (connectionMode) + { + case ConnectionMode.Gateway: + return TestCommon.CreateCosmosClient(true); + case ConnectionMode.Direct: + return TestCommon.CreateCosmosClient(false); + default: + throw new ArgumentException($"Unexpected connection mode: {connectionMode}"); + } + } + + private static async Task> QueryWithContinuationTokens( + CosmosContainer container, + string query, + int maxItemCount, + CosmosQueryRequestOptions queryRequestOptions = null) + { + List results = new List(); + string continuationToken = null; + do + { + CosmosResultSetIterator itemQuery = container.Items.CreateItemQuery( + sqlQueryText: query, + maxConcurrency: 2, + maxItemCount: maxItemCount, + requestOptions: queryRequestOptions, + continuationToken: continuationToken); + + CosmosQueryResponse cosmosQueryResponse = await itemQuery.FetchNextSetAsync(); + results.AddRange(cosmosQueryResponse); + continuationToken = cosmosQueryResponse.ContinuationToken; + } while (continuationToken != null); + + return results; + } + + private static async Task> QueryWithoutContinuationTokens( + CosmosContainer container, + string query, + int maxItemCount, + CosmosQueryRequestOptions queryRequestOptions = null) + { + List results = new List(); + CosmosResultSetIterator itemQuery = container.Items.CreateItemQuery( + sqlQueryText: query, + maxConcurrency: 2, + requestOptions: queryRequestOptions); + + while (itemQuery.HasMoreResults) + { + results.AddRange(await itemQuery.FetchNextSetAsync()); + } + + return results; + } + + private static async Task NoOp() + { + await Task.Delay(0); + } + + private async Task RandomlyThrowException(Exception exception = null) + { + await CrossPartitionQueryTestsOnePartition.NoOp(); + Random random = new Random(); + if (random.Next(0, 2) == 0) + { + throw exception; + } + } + + [TestMethod] + public async Task TestBadQueriesOverMultiplePartitions() + { + await this.CreateIngestQueryDelete( + ConnectionModes.Direct | ConnectionModes.Gateway, + CrossPartitionQueryTestsOnePartition.NoDocuments, + this.TestBadQueriesOverMultiplePartitionsHelper); + } + + [TestMethod] + public void TestContinuationTokenSerialization() + { + CompositeContinuationToken compositeContinuationToken = new CompositeContinuationToken() + { + Token = "asdf", + Range = new Range("asdf", "asdf", false, false), + }; + + string serializedCompositeContinuationToken = JsonConvert.SerializeObject(compositeContinuationToken); + CompositeContinuationToken deserializedCompositeContinuationToken = JsonConvert.DeserializeObject(serializedCompositeContinuationToken); + Assert.AreEqual(compositeContinuationToken.Token, deserializedCompositeContinuationToken.Token); + //Assert.IsTrue(compositeContinuationToken.Range.Equals(deserializedCompositeContinuationToken.Range)); + + + string orderByItemSerialized = @"{""item"" : 1337 }"; + byte[] bytes = Encoding.UTF8.GetBytes(orderByItemSerialized); + OrderByItem orderByItem = new OrderByItem(CosmosElement.Create(bytes)); + OrderByContinuationToken orderByContinuationToken = new OrderByContinuationToken( + compositeContinuationToken, + new List { orderByItem }, + "asdf", + 42, + "asdf"); + string serializedOrderByContinuationToken = JsonConvert.SerializeObject(orderByContinuationToken); + OrderByContinuationToken deserializedOrderByContinuationToken = JsonConvert.DeserializeObject(serializedOrderByContinuationToken); + Assert.AreEqual( + orderByContinuationToken.CompositeContinuationToken.Token, + deserializedOrderByContinuationToken.CompositeContinuationToken.Token); + //Assert.IsTrue( + // orderByContinuationToken.CompositeContinuationToken.Range.Equals( + // deserializedOrderByContinuationToken.CompositeContinuationToken.Range)); + Assert.IsTrue(CosmosElementEqualityComparer.Value.Equals(orderByContinuationToken.OrderByItems[0].Item, deserializedOrderByContinuationToken.OrderByItems[0].Item)); + Assert.AreEqual(orderByContinuationToken.Rid, deserializedOrderByContinuationToken.Rid); + Assert.AreEqual(orderByContinuationToken.SkipCount, deserializedOrderByContinuationToken.SkipCount); + } + + private async Task TestBadQueriesOverMultiplePartitionsHelper(CosmosContainer container, IEnumerable documents) + { + await CrossPartitionQueryTestsOnePartition.NoOp(); + try + { + CosmosResultSetIterator resultSetIterator = container.Items.CreateItemQuery( + @"SELECT * FROM Root r WHERE a = 1", + maxConcurrency: 2); + + await resultSetIterator.FetchNextSetAsync(); + + Assert.Fail("Expected CosmosException"); + } + catch (CosmosException e) when (e.StatusCode == HttpStatusCode.BadRequest) + { + if (!e.Message.StartsWith("Message: {\"errors\":[{\"severity\":\"Error\",\"location\":{\"start\":27,\"end\":28},\"code\":\"SC2001\",\"message\":\"Identifier 'a' could not be resolved.\"}]}")) + { + throw e; + } + } + } + + /// + //"SELECT c._ts, c.id, c.TicketNumber, c.PosCustomerNumber, c.CustomerId, c.CustomerUserId, c.ContactEmail, c.ContactPhone, c.StoreCode, c.StoreUid, c.PoNumber, c.OrderPlacedOn, c.OrderType, c.OrderStatus, c.Customer.UserFirstName, c.Customer.UserLastName, c.Customer.Name, c.UpdatedBy, c.UpdatedOn, c.ExpirationDate, c.TotalAmountFROM c ORDER BY c._ts"' created an ArgumentOutofRangeException since ServiceInterop was returning DISP_E_BUFFERTOOSMALL in the case of an invalid query that is also really long. + /// This test case just double checks that you get the appropriate document client exception instead of just failing. + /// + [TestMethod] + public async Task TestQueryCrossParitionPartitionProviderInvalid() + { + await this.CreateIngestQueryDelete( + ConnectionModes.Direct | ConnectionModes.Gateway, + CrossPartitionQueryTestsOnePartition.NoDocuments, + this.TestQueryCrossParitionPartitionProviderInvalidHelper); + } + + private async Task TestQueryCrossParitionPartitionProviderInvalidHelper(CosmosContainer container, IEnumerable documents) + { + await CrossPartitionQueryTestsOnePartition.NoOp(); + try + { + /// note that there is no space before the from clause thus this query should fail + /// '"code":"SC2001","message":"Identifier 'c' could not be resolved."' + string query = "SELECT c._ts, c.id, c.TicketNumber, c.PosCustomerNumber, c.CustomerId, c.CustomerUserId, c.ContactEmail, c.ContactPhone, c.StoreCode, c.StoreUid, c.PoNumber, c.OrderPlacedOn, c.OrderType, c.OrderStatus, c.Customer.UserFirstName, c.Customer.UserLastName, c.Customer.Name, c.UpdatedBy, c.UpdatedOn, c.ExpirationDate, c.TotalAmountFROM c ORDER BY c._ts"; + List expectedValues = new List(); + CosmosResultSetIterator resultSetIterator = container.Items.CreateItemQuery( + query, + maxConcurrency: 0); + + while (resultSetIterator.HasMoreResults) + { + expectedValues.AddRange(await resultSetIterator.FetchNextSetAsync()); + } + + Assert.Fail("Expected to get an exception for this query."); + } + catch (CosmosException e) when (e.StatusCode == HttpStatusCode.BadRequest) + { + } + } + + [TestMethod] + public async Task TestQueryAndReadFeedWithPartitionKey() + { + string[] documents = new[] + { + @"{""id"":""documentId1"",""key"":""A"",""prop"":3,""shortArray"":[{""a"":5}]}", + @"{""id"":""documentId2"",""key"":""A"",""prop"":2,""shortArray"":[{""a"":6}]}", + @"{""id"":""documentId3"",""key"":""A"",""prop"":1,""shortArray"":[{""a"":7}]}", + @"{""id"":""documentId4"",""key"":5,""prop"":3,""shortArray"":[{""a"":5}]}", + @"{""id"":""documentId5"",""key"":5,""prop"":2,""shortArray"":[{""a"":6}]}", + @"{""id"":""documentId6"",""key"":5,""prop"":1,""shortArray"":[{""a"":7}]}", + @"{""id"":""documentId10"",""prop"":3,""shortArray"":[{""a"":5}]}", + @"{""id"":""documentId11"",""prop"":2,""shortArray"":[{""a"":6}]}", + @"{""id"":""documentId12"",""prop"":1,""shortArray"":[{""a"":7}]}", + }; + + await this.CreateIngestQueryDelete( + ConnectionModes.Direct | ConnectionModes.Gateway, + documents, + this.TestQueryAndReadFeedWithPartitionKeyHelper, + "/key"); + } + + private async Task TestQueryAndReadFeedWithPartitionKeyHelper( + CosmosContainer container, + IEnumerable documents) + { + Assert.AreEqual(0, (await this.RunQuery( + container, + @"SELECT * FROM Root r WHERE false", + maxConcurrency: 1)).Count); + + object[] keys = new object[] { "A", 5, Undefined.Value }; + for (int i = 0; i < keys.Length; ++i) + { + List expected = documents.Skip(i * 3).Take(3).Select(doc => doc.Id).ToList(); + string expectedResult = string.Join(",", expected); + // Order-by + expected.Reverse(); + string expectedOrderByResult = string.Join(",", expected); + + List<(string, string)> queries = new List<(string, string)>() + { + ($@"SELECT * FROM Root r WHERE r.id IN (""{expected[0]}"", ""{expected[1]}"", ""{expected[2]}"")", expectedResult), + (@"SELECT * FROM Root r WHERE r.prop BETWEEN 1 AND 3", expectedResult), + (@"SELECT VALUE r FROM Root r JOIN c IN r.shortArray WHERE c.a BETWEEN 5 and 7", expectedResult), + ($@"SELECT TOP 10 * FROM Root r WHERE r.id IN (""{expected[0]}"", ""{expected[1]}"", ""{expected[2]}"")", expectedResult), + (@"SELECT TOP 10 * FROM Root r WHERE r.prop BETWEEN 1 AND 3", expectedResult), + (@"SELECT TOP 10 VALUE r FROM Root r JOIN c IN r.shortArray WHERE c.a BETWEEN 5 and 7", expectedResult), + ($@"SELECT * FROM Root r WHERE r.id IN (""{expected[0]}"", ""{expected[1]}"", ""{expected[2]}"") ORDER BY r.prop", expectedOrderByResult), + (@"SELECT * FROM Root r WHERE r.prop BETWEEN 1 AND 3 ORDER BY r.prop", expectedOrderByResult), + (@"SELECT VALUE r FROM Root r JOIN c IN r.shortArray WHERE c.a BETWEEN 5 and 7 ORDER BY r.prop", expectedOrderByResult), + }; + + + + if (i < keys.Length - 1) + { + string key; + if (keys[i] is string) + { + key = "'" + keys[i].ToString() + "'"; + } + else + { + key = keys[i].ToString(); + } + + queries.Add((string.Format(CultureInfo.InvariantCulture, @"SELECT * FROM Root r WHERE r.key = {0} ORDER BY r.prop", key), expectedOrderByResult)); + } + + foreach ((string, string) queryAndExpectedResult in queries) + { + CosmosResultSetIterator resultSetIterator = container.Items.CreateItemQuery( + sqlQueryText: queryAndExpectedResult.Item1, + partitionKey: keys[i], + maxItemCount: 1); + + List result = new List(); + while (resultSetIterator.HasMoreResults) + { + result.AddRange(await resultSetIterator.FetchNextSetAsync()); + } + + string resultDocIds = string.Join(",", result.Select(doc => doc.Id)); + Assert.AreEqual(queryAndExpectedResult.Item2, resultDocIds); + } + } + } + + [TestMethod] + public async Task TestQueryMultiplePartitionsSinglePartitionKey() + { + string[] documents = new[] + { + @"{""pk"":""doc1""}", + @"{""pk"":""doc2""}", + @"{""pk"":""doc3""}", + @"{""pk"":""doc4""}", + @"{""pk"":""doc5""}", + @"{""pk"":""doc6""}", + }; + + await this.CreateIngestQueryDelete( + ConnectionModes.Direct | ConnectionModes.Gateway, + documents, + this.TestQueryMultiplePartitionsSinglePartitionKeyHelper, + "/pk"); + } + + private async Task TestQueryMultiplePartitionsSinglePartitionKeyHelper(CosmosContainer container, IEnumerable documents) + { + // Query with partition key should be done in one round trip. + CosmosResultSetIterator resultSetIterator = container.Items.CreateItemQuery( + "SELECT * FROM c WHERE c.pk = 'doc5'", + partitionKey: "doc5"); + + CosmosQueryResponse response = await resultSetIterator.FetchNextSetAsync(); + Assert.AreEqual(1, response.Count()); + Assert.IsNull(response.ContinuationToken); + + resultSetIterator = container.Items.CreateItemQuery( + "SELECT * FROM c WHERE c.pk = 'doc10'", + partitionKey: "doc10"); + + response = await resultSetIterator.FetchNextSetAsync(); + Assert.AreEqual(0, response.Count()); + Assert.IsNull(response.ContinuationToken); + } + + private struct QueryWithSpecialPartitionKeysArgs + { + public string Name; + public object Value; + public Func ValueToPartitionKey; + } + + // V3 only supports Numeric, string, bool, null, undefined + [TestMethod] + [Ignore] + public async Task TestQueryWithSpecialPartitionKeys() + { + await CrossPartitionQueryTestsOnePartition.NoOp(); + QueryWithSpecialPartitionKeysArgs[] queryWithSpecialPartitionKeyArgsList = new QueryWithSpecialPartitionKeysArgs[] + { + new QueryWithSpecialPartitionKeysArgs() + { + Name = "Guid", + Value = Guid.NewGuid(), + ValueToPartitionKey = val => val.ToString(), + }, + //new QueryWithSpecialPartitionKeysArgs() + //{ + // Name = "DateTime", + // Value = DateTime.Now, + // ValueToPartitionKey = val => + // { + // string str = JsonConvert.SerializeObject( + // val, + // new JsonSerializerSettings() + // { + // Converters = new List { new IsoDateTimeConverter() } + // }); + // return str.Substring(1, str.Length - 2); + // }, + //}, + new QueryWithSpecialPartitionKeysArgs() + { + Name = "Enum", + Value = HttpStatusCode.OK, + ValueToPartitionKey = val => (int)val, + }, + new QueryWithSpecialPartitionKeysArgs() + { + Name = "CustomEnum", + Value = HttpStatusCode.OK, + ValueToPartitionKey = val => val.ToString(), + }, + new QueryWithSpecialPartitionKeysArgs() + { + Name = "ResourceId", + Value = "testid", + ValueToPartitionKey = val => val, + }, + new QueryWithSpecialPartitionKeysArgs() + { + Name = "CustomDateTime", + Value = new DateTime(2016, 11, 12), + ValueToPartitionKey = val => EpochDateTimeConverter.DateTimeToEpoch((DateTime)val), + }, + }; + + foreach (QueryWithSpecialPartitionKeysArgs testArg in queryWithSpecialPartitionKeyArgsList) + { + // For this test we need to split direct and gateway runs into separate collections, + // since the query callback inserts some documents (thus has side effects). + await this.CreateIngestQueryDelete( + ConnectionModes.Direct, + CrossPartitionQueryTestsOnePartition.NoDocuments, + this.TestQueryWithSpecialPartitionKeysHelper, + testArg, + "/" + testArg.Name); + + await this.CreateIngestQueryDelete( + ConnectionModes.Gateway, + CrossPartitionQueryTestsOnePartition.NoDocuments, + this.TestQueryWithSpecialPartitionKeysHelper, + testArg, + "/" + testArg.Name); + } + } + + private async Task TestQueryWithSpecialPartitionKeysHelper(CosmosContainer container, IEnumerable documents, QueryWithSpecialPartitionKeysArgs testArgs) + { + QueryWithSpecialPartitionKeysArgs args = testArgs; + + SpecialPropertyDocument specialPropertyDocument = new SpecialPropertyDocument + { + id = Guid.NewGuid().ToString() + }; + + specialPropertyDocument.GetType().GetProperty(args.Name).SetValue(specialPropertyDocument, args.Value); + Func getPropertyValueFunction = d => d.GetType().GetProperty(args.Name).GetValue(d); + + CosmosItemResponse response = await container.Items.CreateItemAsync(testArgs.Value, specialPropertyDocument); + dynamic returnedDoc = response.Resource; + Assert.AreEqual(args.Value, getPropertyValueFunction((SpecialPropertyDocument)returnedDoc)); + + PartitionKey key = new PartitionKey(args.ValueToPartitionKey(args.Value)); + response = await container.Items.ReadItemAsync(key, response.Resource.id); + returnedDoc = response.Resource; + Assert.AreEqual(args.Value, getPropertyValueFunction((SpecialPropertyDocument)returnedDoc)); + + returnedDoc = (await this.RunSinglePartitionQuery( + container, + "SELECT * FROM t", + key)).Single(); + + Assert.AreEqual(args.Value, getPropertyValueFunction(returnedDoc)); + + string query; + switch (args.Name) + { + case "Guid": + query = $"SELECT * FROM T WHERE T.Guid = '{(Guid)args.Value}'"; + break; + case "Enum": + query = $"SELECT * FROM T WHERE T.Enum = '{(HttpStatusCode)args.Value}'"; + break; + case "DateTime": + query = $"SELECT * FROM T WHERE T.DateTime = '{(DateTime)args.Value}'"; + break; + case "CustomEnum": + query = $"SELECT * FROM T WHERE T.CustomEnum = '{(HttpStatusCode)args.Value}'"; + break; + case "ResourceId": + query = $"SELECT * FROM T WHERE T.ResourceId = '{(string)args.Value}'"; + break; + case "CustomDateTime": + query = $"SELECT * FROM T WHERE T.CustomDateTime = '{(DateTime)args.Value}'"; + break; + default: + query = null; + break; + } + + returnedDoc = (await container.Items.CreateItemQuery( + query, + partitionKey: args.ValueToPartitionKey, + maxItemCount: 1).FetchNextSetAsync()).First(); + + Assert.AreEqual(args.Value, getPropertyValueFunction(returnedDoc)); + } + + private sealed class SpecialPropertyDocument + { + public string id + { + get; + set; + } + + public Guid Guid + { + get; + set; + } + + [JsonConverter(typeof(IsoDateTimeConverter))] + public DateTime DateTime + { + get; + set; + } + + [JsonConverter(typeof(EpochDateTimeConverter))] + public DateTime CustomDateTime + { + get; + set; + } + + + public HttpStatusCode Enum + { + get; + set; + } + + [JsonConverter(typeof(StringEnumConverter))] + public HttpStatusCode CustomEnum + { + get; + set; + } + + public string ResourceId + { + get; + set; + } + } + + private sealed class EpochDateTimeConverter : JsonConverter + { + public static int DateTimeToEpoch(DateTime dt) + { + if (!dt.Equals(DateTime.MinValue)) + { + DateTime epoch = new DateTime(1970, 1, 1); + TimeSpan epochTimeSpan = dt - epoch; + return (int)epochTimeSpan.TotalSeconds; + } + else + { + return int.MinValue; + } + } + + public override bool CanConvert(Type objectType) + { + return true; + } + + public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer) + { + if (reader.TokenType == JsonToken.None || reader.TokenType == JsonToken.Null) + { + return null; + } + + + if (reader.TokenType != JsonToken.Integer) + { + throw new Exception( + string.Format( + CultureInfo.InvariantCulture, + "Unexpected token parsing date. Expected Integer, got {0}.", + reader.TokenType)); + } + + int seconds = Convert.ToInt32(reader.Value, CultureInfo.InvariantCulture); + return new DateTime(1970, 1, 1).AddSeconds(seconds); + } + + public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer) + { + int seconds; + if (value is DateTime) + { + seconds = DateTimeToEpoch((DateTime)value); + } + else + { + throw new Exception("Expected date object value."); + } + + writer.WriteValue(seconds); + } + } + + private struct QueryCrossPartitionWithLargeNumberOfKeysArgs + { + public int NumberOfDocuments; + public string PartitionKey; + public HashSet ExpectedPartitionKeyValues; + } + + [TestMethod] + public async Task TestQueryCrossPartitionWithLargeNumberOfKeys() + { + int numberOfDocuments = 1000; + string partitionKey = "key"; + HashSet expectedPartitionKeyValues = new HashSet(); + List documents = new List(); + for (int i = 0; i < numberOfDocuments; i++) + { + Document doc = new Document(); + doc.SetPropertyValue(partitionKey, i); + documents.Add(doc.ToString()); + + expectedPartitionKeyValues.Add(i); + } + + Assert.AreEqual(numberOfDocuments, expectedPartitionKeyValues.Count); + + QueryCrossPartitionWithLargeNumberOfKeysArgs args = new QueryCrossPartitionWithLargeNumberOfKeysArgs() + { + NumberOfDocuments = numberOfDocuments, + PartitionKey = partitionKey, + ExpectedPartitionKeyValues = expectedPartitionKeyValues, + }; + + await this.CreateIngestQueryDelete( + ConnectionModes.Direct | ConnectionModes.Gateway, + documents, + this.TestQueryCrossPartitionWithLargeNumberOfKeysHelper, + args, + "/" + partitionKey); + } + + private async Task TestQueryCrossPartitionWithLargeNumberOfKeysHelper(CosmosContainer container, IEnumerable documents, QueryCrossPartitionWithLargeNumberOfKeysArgs args) + { + CosmosSqlQueryDefinition query = new CosmosSqlQueryDefinition( + $"SELECT VALUE r.{args.PartitionKey} FROM r WHERE ARRAY_CONTAINS(@keys, r.{args.PartitionKey})").UseParameter("@keys", args.ExpectedPartitionKeyValues); + + HashSet actualPartitionKeyValues = new HashSet(); + CosmosResultSetIterator documentQuery = container.Items.CreateItemQuery( + sqlQueryDefinition: query, + maxItemCount: -1, + maxConcurrency: 100); + + while (documentQuery.HasMoreResults) + { + CosmosQueryResponse response = await documentQuery.FetchNextSetAsync(); + foreach (int item in response) + { + actualPartitionKeyValues.Add(item); + } + } + + Assert.IsTrue(actualPartitionKeyValues.SetEquals(args.ExpectedPartitionKeyValues)); + } + + [TestMethod] + public async Task TestBasicCrossPartitionQuery() + { + int seed = (int)(DateTime.UtcNow - new DateTime(1970, 1, 1)).TotalSeconds; + uint numberOfDocuments = 100; + QueryOracle.QueryOracleUtil util = new QueryOracle.QueryOracle2(seed); + IEnumerable documents = util.GetDocuments(numberOfDocuments); + + await this.CreateIngestQueryDelete( + ConnectionModes.Direct, + documents, + this.TestBasicCrossPartitionQueryHelper); + } + + private async Task TestBasicCrossPartitionQueryHelper( + CosmosContainer container, + IEnumerable documents) + { + foreach (int maxDegreeOfParallelism in new int[] { 1, 100 }) + { + foreach (int maxItemCount in new int[] { 10, 100 }) + { + CosmosQueryRequestOptions feedOptions = new CosmosQueryRequestOptions + { + EnableCrossPartitionQuery = true, + MaxBufferedItemCount = 7000, + MaxConcurrency = maxDegreeOfParallelism + }; + + List actualFromQueryWithoutContinutionTokens; + actualFromQueryWithoutContinutionTokens = await QueryWithoutContinuationTokens( + container, + "SELECT * FROM c", + maxItemCount, + feedOptions); + + Assert.AreEqual(documents.Count(), actualFromQueryWithoutContinutionTokens.Count); + } + } + } + + [TestMethod] + public async Task TestQueryCrossPartitionAggregateFunctions() + { + AggregateTestArgs aggregateTestArgs = new AggregateTestArgs() + { + NumberOfDocsWithSamePartitionKey = 100, + NumberOfDocumentsDifferentPartitionKey = 100, + PartitionKey = "key", + UniquePartitionKey = "uniquePartitionKey", + Field = "field", + Values = new object[] { false, true, "abc", "cdfg", "opqrs", "ttttttt", "xyz" }, + }; + + List documents = new List(aggregateTestArgs.NumberOfDocumentsDifferentPartitionKey + aggregateTestArgs.NumberOfDocsWithSamePartitionKey); + foreach (object val in aggregateTestArgs.Values) + { + Document doc; + doc = new Document(); + doc.SetPropertyValue(aggregateTestArgs.PartitionKey, val); + doc.SetPropertyValue("id", Guid.NewGuid().ToString()); + + documents.Add(doc.ToString()); + } + + for (int i = 0; i < aggregateTestArgs.NumberOfDocsWithSamePartitionKey; ++i) + { + Document doc = new Document(); + doc.SetPropertyValue(aggregateTestArgs.PartitionKey, aggregateTestArgs.UniquePartitionKey); + doc.ResourceId = i.ToString(CultureInfo.InvariantCulture); + doc.SetPropertyValue(aggregateTestArgs.Field, i + 1); + doc.SetPropertyValue("id", Guid.NewGuid().ToString()); + + documents.Add(doc.ToString()); + } + + for (int i = 0; i < aggregateTestArgs.NumberOfDocumentsDifferentPartitionKey; ++i) + { + Document doc = new Document(); + doc.SetPropertyValue(aggregateTestArgs.PartitionKey, i + 1); + doc.SetPropertyValue("id", Guid.NewGuid().ToString()); + documents.Add(doc.ToString()); + } + + await this.CreateIngestQueryDelete( + ConnectionModes.Direct | ConnectionModes.Gateway, + documents, + this.TestQueryCrossPartitionAggregateFunctionsAsync, + aggregateTestArgs, + "/" + aggregateTestArgs.PartitionKey); + } + + private struct AggregateTestArgs + { + public int NumberOfDocumentsDifferentPartitionKey; + public int NumberOfDocsWithSamePartitionKey; + public string PartitionKey; + public string UniquePartitionKey; + public string Field; + public object[] Values; + } + + private struct AggregateQueryArguments + { + public string AggregateOperator; + public object ExpectedValue; + public string Predicate; + } + + private async Task TestQueryCrossPartitionAggregateFunctionsAsync(CosmosContainer container, IEnumerable documents, AggregateTestArgs aggregateTestArgs) + { + int numberOfDocumentsDifferentPartitionKey = aggregateTestArgs.NumberOfDocumentsDifferentPartitionKey; + int numberOfDocumentSamePartitionKey = aggregateTestArgs.NumberOfDocsWithSamePartitionKey; + int numberOfDocuments = aggregateTestArgs.NumberOfDocumentsDifferentPartitionKey + aggregateTestArgs.NumberOfDocsWithSamePartitionKey; + object[] values = aggregateTestArgs.Values; + string partitionKey = aggregateTestArgs.PartitionKey; + + double samePartitionSum = ((numberOfDocumentSamePartitionKey * (numberOfDocumentSamePartitionKey + 1)) / 2); + double differentPartitionSum = ((numberOfDocumentsDifferentPartitionKey * (numberOfDocumentsDifferentPartitionKey + 1)) / 2); + double partitionSum = samePartitionSum + differentPartitionSum; + AggregateQueryArguments[] aggregateQueryArgumentsList = new AggregateQueryArguments[] + { + new AggregateQueryArguments() + { + AggregateOperator = "AVG", + ExpectedValue = partitionSum / numberOfDocuments, + Predicate = $"IS_NUMBER(r.{partitionKey})", + }, + new AggregateQueryArguments() + { + AggregateOperator = "AVG", + ExpectedValue = Undefined.Value, + Predicate = "true", + }, + new AggregateQueryArguments() + { + AggregateOperator = "COUNT", + ExpectedValue = (long)numberOfDocuments + values.Length, + Predicate = "true", + }, + new AggregateQueryArguments() + { + AggregateOperator = "MAX", + ExpectedValue = "xyz", + Predicate = "true", + }, + new AggregateQueryArguments() + { + AggregateOperator = "MIN", + ExpectedValue = false, + Predicate = "true", + }, + new AggregateQueryArguments() + { + AggregateOperator = "SUM", + ExpectedValue = differentPartitionSum, + Predicate = $"IS_NUMBER(r.{partitionKey})", + }, + new AggregateQueryArguments() + { + AggregateOperator = "SUM", + ExpectedValue = Undefined.Value, + Predicate = $"true", + }, + }; + + foreach (int maxDoP in new[] { 0, 10 }) + { + foreach (AggregateQueryArguments argument in aggregateQueryArgumentsList) + { + string[] queryFormats = new[] + { + "SELECT VALUE {0}(r.{1}) FROM r WHERE {2}", + "SELECT VALUE {0}(r.{1}) FROM r WHERE {2} ORDER BY r.{1}" + }; + + foreach (string queryFormat in queryFormats) + { + string query = string.Format(CultureInfo.InvariantCulture, queryFormat, argument.AggregateOperator, partitionKey, argument.Predicate); + string message = string.Format(CultureInfo.InvariantCulture, "query: {0}, data: {1}", query, JsonConvert.SerializeObject(argument)); + List items = new List(); + + CosmosResultSetIterator resultSetIterator = container.Items.CreateItemQuery( + query, + maxConcurrency: maxDoP); + while (resultSetIterator.HasMoreResults) + { + items.AddRange(await resultSetIterator.FetchNextSetAsync()); + } + + if (Undefined.Value.Equals(argument.ExpectedValue)) + { + Assert.AreEqual(0, items.Count, message); + } + else + { + object expected = argument.ExpectedValue; + object actual = items.Single(); + + if (expected is long) + { + expected = (double)((long)expected); + } + + if (actual is long) + { + actual = (double)((long)actual); + } + + Assert.AreEqual(expected, actual, message); + } + } + } + + // Single partition queries + double singlePartitionSum = samePartitionSum; + Tuple[] datum = new[] + { + Tuple.Create("AVG", singlePartitionSum / numberOfDocumentSamePartitionKey), + Tuple.Create("COUNT", (long)numberOfDocumentSamePartitionKey), + Tuple.Create("MAX", (long)numberOfDocumentSamePartitionKey), + Tuple.Create("MIN", (long)1), + Tuple.Create("SUM", (long)singlePartitionSum), + }; + + string field = aggregateTestArgs.Field; + string uniquePartitionKey = aggregateTestArgs.UniquePartitionKey; + foreach (Tuple data in datum) + { + string query = $"SELECT VALUE {data.Item1}(r.{field}) FROM r WHERE r.{partitionKey} = '{uniquePartitionKey}'"; + dynamic aggregate = (await QueryWithoutContinuationTokens(container, query, maxItemCount: 1)).Single(); + object expected = data.Item2; + + if (aggregate is long) + { + aggregate = (long)aggregate; + } + + if (expected is long) + { + expected = (long)expected; + } + + Assert.AreEqual( + expected, + aggregate, + string.Format(CultureInfo.InvariantCulture, "query: {0}, data: {1}", query, JsonConvert.SerializeObject(data))); + + // V3 doesn't support an equivalent to ToList() + // Aggregate queries need to be in the form SELECT VALUE + //query = $"SELECT {data.Item1}(r.{field}) FROM r WHERE r.{partitionKey} = '{uniquePartitionKey}'"; + //try + //{ + // documentClient.CreateDocumentQuery( + // collection, + // query).ToList().Single(); + // Assert.Fail($"Expect exception query: {query}"); + //} + //catch (AggregateException ex) + //{ + // if (!(ex.InnerException is CosmosException) || ((CosmosException)ex.InnerException).StatusCode != HttpStatusCode.BadRequest) + // { + // throw; + // } + //} + + // Make sure ExecuteNextAsync works for unsupported aggregate projection + CosmosQueryResponse page = await container.Items.CreateItemQuery(query, maxConcurrency: 1).FetchNextSetAsync(); + } + } + } + + [TestMethod] + public async Task TestQueryCrossPartitionAggregateFunctionsEmptyPartitions() + { + AggregateQueryEmptyPartitionsArgs args = new AggregateQueryEmptyPartitionsArgs() + { + NumDocuments = 100, + PartitionKey = "key", + UniqueField = "UniqueField", + }; + + List documents = new List(args.NumDocuments); + for (int i = 0; i < args.NumDocuments; ++i) + { + Document doc = new Document(); + doc.SetPropertyValue(args.PartitionKey, Guid.NewGuid()); + doc.SetPropertyValue(args.UniqueField, i); + documents.Add(doc.ToString()); + } + + await this.CreateIngestQueryDelete( + ConnectionModes.Direct | ConnectionModes.Gateway, + documents, + this.TestQueryCrossPartitionAggregateFunctionsEmptyPartitionsHelper, + args, + "/" + args.PartitionKey); + } + + private struct AggregateQueryEmptyPartitionsArgs + { + public int NumDocuments; + public string PartitionKey; + public string UniqueField; + } + + private async Task TestQueryCrossPartitionAggregateFunctionsEmptyPartitionsHelper(CosmosContainer container, IEnumerable documents, AggregateQueryEmptyPartitionsArgs args) + { + await CrossPartitionQueryTestsOnePartition.NoOp(); + int numDocuments = args.NumDocuments; + string partitionKey = args.PartitionKey; + string uniqueField = args.UniqueField; + + // Perform full fanouts but only match a single value that isn't the partition key. + // This leads to all other partitions returning { "" = UNDEFINDED, "count" = 0 } + // which should be ignored from the aggregation. + int valueOfInterest = args.NumDocuments / 2; + string[] queries = new string[] + { + $"SELECT VALUE AVG(c.{uniqueField}) FROM c WHERE c.{uniqueField} = {valueOfInterest}", + $"SELECT VALUE MIN(c.{uniqueField}) FROM c WHERE c.{uniqueField} = {valueOfInterest}", + $"SELECT VALUE MAX(c.{uniqueField}) FROM c WHERE c.{uniqueField} = {valueOfInterest}", + $"SELECT VALUE SUM(c.{uniqueField}) FROM c WHERE c.{uniqueField} = {valueOfInterest}", + }; + + foreach (string query in queries) + { + try + { + List items = await this.RunQuery( + container, + query, + maxConcurrency: 10); + + Assert.AreEqual(valueOfInterest, items.Single()); + } + catch (Exception ex) + { + Assert.Fail($"Something went wrong with query: {query}, ex: {ex}"); + } + } + } + + [TestMethod] + public async Task TestQueryCrossPartitionAggregateFunctionsWithMixedTypes() + { + AggregateQueryMixedTypes args = new AggregateQueryMixedTypes() + { + PartitionKey = "key", + Field = "field", + DoubleOnlyKey = "doubleOnly", + StringOnlyKey = "stringOnly", + BoolOnlyKey = "boolOnly", + NullOnlyKey = "nullOnly", + ObjectOnlyKey = "objectOnlyKey", + ArrayOnlyKey = "arrayOnlyKey", + OneObjectKey = "oneObjectKey", + OneArrayKey = "oneArrayKey", + UndefinedKey = "undefinedKey", + }; + + List documents = new List(); + Random random = new Random(1234); + for (int i = 0; i < 20; ++i) + { + Document doubleDoc = new Document(); + doubleDoc.SetPropertyValue(args.PartitionKey, Guid.NewGuid()); + doubleDoc.SetPropertyValue(args.Field, random.Next(1, 100000)); + documents.Add(doubleDoc.ToString()); + doubleDoc.SetPropertyValue(args.PartitionKey, args.DoubleOnlyKey); + documents.Add(doubleDoc.ToString()); + + Document stringDoc = new Document(); + stringDoc.SetPropertyValue(args.PartitionKey, Guid.NewGuid()); + stringDoc.SetPropertyValue(args.Field, random.NextDouble().ToString()); + documents.Add(stringDoc.ToString()); + stringDoc.SetPropertyValue(args.PartitionKey, args.StringOnlyKey); + documents.Add(stringDoc.ToString()); + + Document boolDoc = new Document(); + boolDoc.SetPropertyValue(args.PartitionKey, Guid.NewGuid()); + boolDoc.SetPropertyValue(args.Field, random.Next() % 2 == 0); + documents.Add(boolDoc.ToString()); + boolDoc.SetPropertyValue(args.PartitionKey, args.BoolOnlyKey); + documents.Add(boolDoc.ToString()); + + Document nullDoc = new Document(); + nullDoc.SetPropertyValue(args.PartitionKey, Guid.NewGuid()); + nullDoc.propertyBag.Add(args.Field, null); + documents.Add(nullDoc.ToString()); + nullDoc.SetPropertyValue(args.PartitionKey, args.NullOnlyKey); + documents.Add(nullDoc.ToString()); + + Document objectDoc = new Document(); + objectDoc.SetPropertyValue(args.PartitionKey, Guid.NewGuid()); + objectDoc.SetPropertyValue(args.Field, new object { }); + documents.Add(objectDoc.ToString()); + objectDoc.SetPropertyValue(args.PartitionKey, args.ObjectOnlyKey); + documents.Add(objectDoc.ToString()); + + Document arrayDoc = new Document(); + arrayDoc.SetPropertyValue(args.PartitionKey, Guid.NewGuid()); + arrayDoc.SetPropertyValue(args.Field, new object[] { }); + documents.Add(arrayDoc.ToString()); + arrayDoc.SetPropertyValue(args.PartitionKey, args.ArrayOnlyKey); + documents.Add(arrayDoc.ToString()); + } + + Document oneObjectDoc = new Document(); + oneObjectDoc.SetPropertyValue(args.PartitionKey, args.OneObjectKey); + oneObjectDoc.SetPropertyValue(args.Field, new object { }); + documents.Add(oneObjectDoc.ToString()); + + Document oneArrayDoc = new Document(); + oneArrayDoc.SetPropertyValue(args.PartitionKey, args.OneArrayKey); + oneArrayDoc.SetPropertyValue(args.Field, new object[] { }); + documents.Add(oneArrayDoc.ToString()); + + Document undefinedDoc = new Document(); + undefinedDoc.SetPropertyValue(args.PartitionKey, args.UndefinedKey); + // This doc does not have the field key set + documents.Add(undefinedDoc.ToString()); + + await this.CreateIngestQueryDelete( + ConnectionModes.Direct, + documents, + this.TestQueryCrossPartitionAggregateFunctionsWithMixedTypesHelper, + args, + "/" + args.PartitionKey); + } + + private struct AggregateQueryMixedTypes + { + public string PartitionKey; + public string Field; + public string DoubleOnlyKey; + public string StringOnlyKey; + public string BoolOnlyKey; + public string NullOnlyKey; + public string ObjectOnlyKey; + public string ArrayOnlyKey; + public string OneObjectKey; + public string OneArrayKey; + public string UndefinedKey; + } + + private async Task TestQueryCrossPartitionAggregateFunctionsWithMixedTypesHelper( + CosmosContainer container, + IEnumerable documents, + AggregateQueryMixedTypes args) + { + await CrossPartitionQueryTestsOnePartition.NoOp(); + string partitionKey = args.PartitionKey; + string field = args.Field; + string[] typeOnlyPartitionKeys = new string[] + { + args.DoubleOnlyKey, + args.StringOnlyKey, + args.BoolOnlyKey, + args.NullOnlyKey, + args.ObjectOnlyKey, + args.ArrayOnlyKey, + args.OneArrayKey, + args.OneObjectKey, + args.UndefinedKey + }; + + string[] aggregateOperators = new string[] { "AVG", "MIN", "MAX", "SUM", "COUNT" }; + string[] typeCheckFunctions = new string[] { "IS_ARRAY", "IS_BOOL", "IS_NULL", "IS_NUMBER", "IS_OBJECT", "IS_STRING", "IS_DEFINED", "IS_PRIMITIVE" }; + List queries = new List(); + foreach (string aggregateOperator in aggregateOperators) + { + foreach (string typeCheckFunction in typeCheckFunctions) + { + queries.Add( + $@" + SELECT VALUE {aggregateOperator} (c.{field}) + FROM c + WHERE {typeCheckFunction}(c.{field}) + "); + } + + foreach (string typeOnlyPartitionKey in typeOnlyPartitionKeys) + { + queries.Add( + $@" + SELECT VALUE {aggregateOperator} (c.{field}) + FROM c + WHERE c.{partitionKey} = ""{typeOnlyPartitionKey}"" + "); + } + }; + + // mixing primitive and non primitives + foreach (string minmaxop in new string[] { "MIN", "MAX" }) + { + foreach (string key in new string[] { args.OneObjectKey, args.OneArrayKey }) + { + queries.Add( + $@" + SELECT VALUE {minmaxop} (c.{field}) + FROM c + WHERE c.{partitionKey} IN (""{key}"", ""{args.DoubleOnlyKey}"") + "); + } + } + + + string filename = $"CrossPartitionQueryTests.AggregateMixedTypes"; + string outputPath = $"{filename}_output.xml"; + string baselinePath = $"{filename}_baseline.xml"; + XmlWriterSettings settings = new XmlWriterSettings() + { + OmitXmlDeclaration = true, + Indent = true, + NewLineOnAttributes = true, + }; + using (XmlWriter writer = XmlWriter.Create(outputPath, settings)) + { + writer.WriteStartDocument(); + writer.WriteStartElement("Results"); + foreach (string query in queries) + { + string formattedQuery = string.Join( + Environment.NewLine, + query.Trim().Split( + new[] { Environment.NewLine }, + StringSplitOptions.None) + .Select(x => x.Trim())); + + List items = await this.RunQuery( + container, + query, + 10, + null); + + writer.WriteStartElement("Result"); + writer.WriteStartElement("Query"); + writer.WriteCData(formattedQuery); + writer.WriteEndElement(); + writer.WriteStartElement("Aggregation"); + if (items.Count > 0) + { + writer.WriteCData(JsonConvert.SerializeObject(items.Single())); + } + writer.WriteEndElement(); + writer.WriteEndElement(); + } + writer.WriteEndElement(); + writer.WriteEndDocument(); + } + + Regex r = new Regex(">\\s+"); + string normalizedBaseline = r.Replace(File.ReadAllText(baselinePath), ">"); + string normalizedOutput = r.Replace(File.ReadAllText(outputPath), ">"); + + Assert.AreEqual(normalizedBaseline, normalizedOutput); + } + + [TestMethod] + public async Task TestQueryDistinct() + { + int seed = (int)(DateTime.UtcNow - new DateTime(1970, 1, 1)).TotalSeconds; + uint numberOfDocuments = 100; + + Random rand = new Random(seed); + List people = new List(); + + for (int i = 0; i < numberOfDocuments; i++) + { + Person person = CrossPartitionQueryTestsOnePartition.GetRandomPerson(rand); + for (int j = 0; j < rand.Next(0, 4); j++) + { + people.Add(person); + } + } + + List documents = new List(); + people = people.OrderBy((person) => Guid.NewGuid()).ToList(); + foreach (Person person in people) + { + documents.Add(JsonConvert.SerializeObject(person)); + } + + await this.CreateIngestQueryDelete( + ConnectionModes.Direct | ConnectionModes.Gateway, + documents, + this.TestQueryDistinct, + "/id"); + } + + private async Task TestQueryDistinct(CosmosContainer container, IEnumerable documents, dynamic testArgs = null) + { + #region Queries + // To verify distint queries you can run it once without the distinct clause and run it through a hash set + // then compare to the query with the distinct clause. + List queries = new List() + { + // basic distinct queries + "SELECT {0} VALUE null", + "SELECT {0} VALUE false", + "SELECT {0} VALUE true", + "SELECT {0} VALUE 1", + "SELECT {0} VALUE 'a'", + "SELECT {0} VALUE [null, true, false, 1, 'a']", + "SELECT {0} VALUE {{p1:null, p2:true, p3:false, p4:1, p5:'a'}}", + "SELECT {0} false AS p", + "SELECT {0} 1 AS p", + "SELECT {0} 'a' AS p", + "SELECT {0} [null, true, false, 1, 'a'] AS p", + "SELECT {0} {{p1:null, p2:true, p3:false, p4:1, p5:'a'}} AS p", + "SELECT {0} VALUE {{p1:null, p2:true, p3:false, p4:1, p5:'a'}}", + "SELECT {0} VALUE null FROM c", + "SELECT {0} VALUE false FROM c", + "SELECT {0} VALUE 1 FROM c", + "SELECT {0} VALUE 'a' FROM c", + "SELECT {0} VALUE [null, true, false, 1, 'a'] FROM c", + "SELECT {0} null AS p FROM c", + "SELECT {0} false AS p FROM c", + "SELECT {0} 1 AS p FROM c", + "SELECT {0} 'a' AS p FROM c", + "SELECT {0} [null, true, false, 1, 'a'] AS p FROM c", + "SELECT {0} {{p1:null, p2:true, p3:false, p4:1, p5:'a'}} AS p FROM c", + + // number value distinct queries + "SELECT {0} VALUE c.income from c", + "SELECT {0} VALUE c.age from c", + "SELECT {0} c.income, c.income AS income2 from c", + "SELECT {0} c.income, c.age from c", + "SELECT {0} VALUE [c.income, c.age] from c", + + // string value distinct queries + "SELECT {0} VALUE c.name from c", + "SELECT {0} VALUE c.city from c", + "SELECT {0} VALUE c.partitionKey from c", + "SELECT {0} c.name, c.name AS name2 from c", + "SELECT {0} c.name, c.city from c", + "SELECT {0} VALUE [c.name, c.city] from c", + + // array value distinct queries + "SELECT {0} VALUE c.children from c", + "SELECT {0} c.children, c.children AS children2 from c", + "SELECT {0} VALUE [c.name, c.age, c.pet] from c", + + // object value distinct queries + "SELECT {0} VALUE c.pet from c", + "SELECT {0} c.pet, c.pet AS pet2 from c", + + // scalar expressions distinct query + "SELECT {0} VALUE c.age % 2 FROM c", + "SELECT {0} VALUE ABS(c.age) FROM c", + "SELECT {0} VALUE LEFT(c.name, 1) FROM c", + "SELECT {0} VALUE c.name || ', ' || (c.city ?? '') FROM c", + "SELECT {0} VALUE ARRAY_LENGTH(c.children) FROM c", + "SELECT {0} VALUE IS_DEFINED(c.city) FROM c", + "SELECT {0} VALUE (c.children[0].age ?? 0) + (c.children[1].age ?? 0) FROM c", + + // distinct queries with order by + "SELECT {0} VALUE c.age FROM c ORDER BY c.age", + "SELECT {0} VALUE c.name FROM c ORDER BY c.name", + "SELECT {0} VALUE c.city FROM c ORDER BY c.city", + "SELECT {0} VALUE c.city FROM c ORDER BY c.age", + "SELECT {0} VALUE LEFT(c.name, 1) FROM c ORDER BY c.name", + + // distinct queries with top and no matching order by + "SELECT {0} TOP 2147483647 VALUE c.age FROM c", + + // distinct queries with top and matching order by + "SELECT {0} TOP 2147483647 VALUE c.age FROM c ORDER BY c.age", + + // distinct queries with aggregates + "SELECT {0} VALUE MAX(c.age) FROM c", + + // distinct queries with joins + "SELECT {0} VALUE c.age FROM p JOIN c IN p.children", + "SELECT {0} p.age AS ParentAge, c.age ChildAge FROM p JOIN c IN p.children", + "SELECT {0} VALUE c.name FROM p JOIN c IN p.children", + "SELECT {0} p.name AS ParentName, c.name ChildName FROM p JOIN c IN p.children", + + // distinct queries in subqueries + "SELECT {0} r.age, s FROM r JOIN (SELECT DISTINCT VALUE c FROM (SELECT 1 a) c) s WHERE r.age > 25", + "SELECT {0} p.name, p.age FROM (SELECT DISTINCT * FROM r) p WHERE p.age > 25", + + // distinct queries in scalar subqeries + "SELECT {0} p.name, (SELECT DISTINCT VALUE p.age) AS Age FROM p", + "SELECT {0} p.name, p.age FROM p WHERE (SELECT DISTINCT VALUE LEFT(p.name, 1)) > 'A' AND (SELECT DISTINCT VALUE p.age) > 21", + "SELECT {0} p.name, (SELECT DISTINCT VALUE p.age) AS Age FROM p WHERE (SELECT DISTINCT VALUE p.name) > 'A' OR (SELECT DISTINCT VALUE p.age) > 21", + + // select * + "SELECT {0} * FROM c", + }; + #endregion + #region ExecuteNextAsync API + // run the query with distinct and without + MockDistinctMap + // Should receive same results + // PageSize = 1 guarantees that the backend will return some duplicates. + foreach (string query in queries) + { + foreach (int pageSize in new int[] { 1, 10, 100 }) + { + string queryWithDistinct = string.Format(query, "DISTINCT"); + string queryWithoutDistinct = string.Format(query, ""); + MockDistinctMap documentsSeen = new MockDistinctMap(); + List documentsFromWithDistinct = new List(); + List documentsFromWithoutDistinct = new List(); + + CosmosResultSetIterator documentQueryWithoutDistinct = container.Items.CreateItemQuery( + queryWithoutDistinct, + maxConcurrency: 100, + maxItemCount: pageSize); + + while (documentQueryWithoutDistinct.HasMoreResults) + { + CosmosQueryResponse cosmosQueryResponse = await documentQueryWithoutDistinct.FetchNextSetAsync(); + foreach (JToken document in cosmosQueryResponse) + { + if (documentsSeen.Add(document, out UInt192? hash)) + { + documentsFromWithoutDistinct.Add(document); + } + else + { + // No Op for debugging purposes. + } + } + } + + CosmosResultSetIterator documentQueryWithDistinct = container.Items.CreateItemQuery( + queryWithDistinct, + maxConcurrency: 100, + maxItemCount: pageSize); + + while (documentQueryWithDistinct.HasMoreResults) + { + CosmosQueryResponse cosmosQueryResponse = await documentQueryWithDistinct.FetchNextSetAsync(); + documentsFromWithDistinct.AddRange(cosmosQueryResponse); + } + + try + { + Assert.AreEqual(documentsFromWithDistinct.Count, documentsFromWithoutDistinct.Count()); + for (int i = 0; i < documentsFromWithDistinct.Count; i++) + { + JToken documentFromWithDistinct = documentsFromWithDistinct.ElementAt(i); + JToken documentFromWithoutDistinct = documentsFromWithoutDistinct.ElementAt(i); + Assert.IsTrue( + JsonTokenEqualityComparer.Value.Equals(documentFromWithDistinct, documentFromWithoutDistinct), + $"{documentFromWithDistinct} did not match {documentFromWithoutDistinct} at index {i} for {queryWithDistinct}, with page size: {pageSize} on a container"); + } + } + catch (Exception e) + { + throw e; + } + } + } + #endregion + #region Unordered Continuation + // Run the unordered distinct query through the continuation api should result in the same set(but maybe some duplicates) + foreach (string query in new string[] + { + "SELECT {0} VALUE c.name from c", + "SELECT {0} VALUE c.age from c", + "SELECT {0} TOP 2147483647 VALUE c.city from c", + "SELECT {0} VALUE c.age from c ORDER BY c.name", + }) + { + string queryWithDistinct = string.Format(query, "DISTINCT"); + string queryWithoutDistinct = string.Format(query, ""); + HashSet documentsFromWithDistinct = new HashSet(JsonTokenEqualityComparer.Value); + HashSet documentsFromWithoutDistinct = new HashSet(JsonTokenEqualityComparer.Value); + + CosmosResultSetIterator documentQueryWithoutDistinct = container.Items.CreateItemQuery( + queryWithoutDistinct, + maxItemCount: 10, + maxConcurrency: 100); + + while (documentQueryWithoutDistinct.HasMoreResults) + { + CosmosQueryResponse cosmosQueryResponse = await documentQueryWithoutDistinct.FetchNextSetAsync(); + foreach (JToken jToken in cosmosQueryResponse) + { + documentsFromWithoutDistinct.Add(jToken); + } + } + + CosmosResultSetIterator documentQueryWithDistinct = container.Items.CreateItemQuery( + queryWithDistinct, + maxItemCount: 10, + maxConcurrency: 100); + + // For now we are blocking the use of continuation + // This try catch can be removed if we do allow the continuation token. + try + { + string continuationToken = null; + do + { + CosmosResultSetIterator documentQuery = container.Items.CreateItemQuery( + queryWithDistinct, + maxItemCount: 10, + maxConcurrency: 100); + + CosmosQueryResponse cosmosQueryResponse = await documentQuery.FetchNextSetAsync(); + foreach (JToken jToken in cosmosQueryResponse) + { + documentsFromWithDistinct.Add(jToken); + } + + continuationToken = cosmosQueryResponse.ContinuationToken; + + } + while (continuationToken != null); + Assert.IsTrue( + documentsFromWithDistinct.IsSubsetOf(documentsFromWithoutDistinct), + $"Documents didn't match for {queryWithDistinct} on a Partitioned container"); + + Assert.Fail("Expected an exception when using continuation tokens on an unordered distinct query."); + } + catch (ArgumentException ex) + { + string disallowContinuationErrorMessage = RMResources.UnorderedDistinctQueryContinuationToken; + Assert.AreEqual(disallowContinuationErrorMessage, ex.Message); + } + } + #endregion + #region Ordered Region + // Run the ordered distinct query through the continuation api, should result in the same set + // since the previous hash is passed in the continuation token. + foreach (string query in new string[] + { + "SELECT {0} VALUE c.age FROM c ORDER BY c.age", + "SELECT {0} VALUE c.name FROM c ORDER BY c.name", + }) + { + foreach (int pageSize in new int[] { 1, 10, 100 }) + { + string queryWithDistinct = string.Format(query, "DISTINCT"); + string queryWithoutDistinct = string.Format(query, ""); + MockDistinctMap documentsSeen = new MockDistinctMap(); + List documentsFromWithDistinct = new List(); + List documentsFromWithoutDistinct = new List(); + + CosmosResultSetIterator documentQueryWithoutDistinct = container.Items.CreateItemQuery( + sqlQueryText: queryWithoutDistinct, + maxConcurrency: 100, + maxItemCount: 1); + + while (documentQueryWithoutDistinct.HasMoreResults) + { + CosmosQueryResponse cosmosQueryResponse = await documentQueryWithoutDistinct.FetchNextSetAsync(); + foreach (JToken document in cosmosQueryResponse) + { + if (documentsSeen.Add(document, out UInt192? hash)) + { + documentsFromWithoutDistinct.Add(document); + } + else + { + // No Op for debugging purposes. + } + } + } + + CosmosResultSetIterator documentQueryWithDistinct = container.Items.CreateItemQuery( + sqlQueryText: queryWithDistinct, + maxConcurrency: 100, + maxItemCount: 1); + + string continuationToken = null; + do + { + CosmosResultSetIterator cosmosQuery = container.Items.CreateItemQuery( + sqlQueryText: queryWithDistinct, + maxConcurrency: 100, + maxItemCount: 1, + continuationToken: continuationToken); + + CosmosQueryResponse cosmosQueryResponse = await cosmosQuery.FetchNextSetAsync(); + documentsFromWithDistinct.AddRange(cosmosQueryResponse); + continuationToken = cosmosQueryResponse.ContinuationToken; + } + while (continuationToken != null); + + Assert.IsTrue( + documentsFromWithDistinct.SequenceEqual(documentsFromWithoutDistinct, JsonTokenEqualityComparer.Value), + $"Documents didn't match for {queryWithDistinct} on a Partitioned container"); + } + } + #endregion + } + + [TestMethod] + public async Task TestQueryCrossPartitionTopOrderByDifferentDimension() + { + string[] documents = new[] + { + @"{""id"":""documentId1"",""key"":""A""}", + @"{""id"":""documentId2"",""key"":""A"",""prop"":3}", + @"{""id"":""documentId3"",""key"":""A""}", + @"{""id"":""documentId4"",""key"":5}", + @"{""id"":""documentId5"",""key"":5,""prop"":2}", + @"{""id"":""documentId6"",""key"":5}", + @"{""id"":""documentId7"",""key"":2}", + @"{""id"":""documentId8"",""key"":2,""prop"":1}", + @"{""id"":""documentId9"",""key"":2}", + }; + + await this.CreateIngestQueryDelete( + ConnectionModes.Direct | ConnectionModes.Gateway, + documents, + this.TestQueryCrossPartitionTopOrderByDifferentDimensionHelper, + "/key"); + } + + private async Task TestQueryCrossPartitionTopOrderByDifferentDimensionHelper(CosmosContainer container, IEnumerable documents) + { + await CrossPartitionQueryTestsOnePartition.NoOp(); + + string[] expected = new[] { "documentId2", "documentId5", "documentId8" }; + List query = await this.RunQuery( + container, + "SELECT r.id FROM r ORDER BY r.prop DESC", + maxItemCount: 1, + maxConcurrency: 1); + + Assert.AreEqual(string.Join(", ", expected), string.Join(", ", query.Select(doc => doc.Id))); + } + + [TestMethod] + public async Task TestOrderByNonAsciiCharacters() + { + string[] specialStrings = new string[] + { + // Strings which may be used elsewhere in code + "undefined", + // Numeric Strings + "-9223372036854775808/-1", + // Non-whitespace C0 controls: U+0001 through U+0008, U+000E through U+001F, + "\u0001", + // "Byte order marks" + "U+FEFF", + // Unicode Symbols + "ЁЂЃЄЅІЇЈЉЊЋЌЍЎЏАБВГДЕЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯабвгдежзийклмнопрстуфхцчшщъыьэюя", + // Quotation Marks + "", + // Strings which contain two-byte characters: can cause rendering issues or character-length issues + "찦차를 타고 온 펲시맨과 쑛다리 똠방각하", + // Changing length when lowercased + "Ⱥ", + // Japanese Emoticons + "ﾟ･✿ヾ╲(｡◕‿◕｡)╱✿･ﾟ", + // Emoji + "❤️ 💔 💌 💕 💞 💓 💗 💖 💘 💝 💟 💜 💛 💚 💙", + // Strings which contain "corrupted" text. The corruption will not appear in non-HTML text, however. (via http://www.eeemo.net) + "Ṱ̺̺̕o͞ ̷i̲̬͇̪͙n̝̗͕v̟̜̘̦͟o̶̙̰̠kè͚̮̺̪̹̱̤ ̖t̝͕̳̣̻̪͞h̼͓̲̦̳̘̲e͇̣̰̦̬͎ ̢̼̻̱̘h͚͎͙̜̣̲ͅi̦̲̣̰̤v̻͍e̺̭̳̪̰-m̢iͅn̖̺̞̲̯̰d̵̼̟͙̩̼̘̳ ̞̥̱̳̭r̛̗̘e͙p͠r̼̞̻̭̗e̺̠̣͟s̘͇̳͍̝͉e͉̥̯̞̲͚̬͜ǹ̬͎͎̟̖͇̤t͍̬̤͓̼̭͘ͅi̪̱n͠g̴͉ ͏͉ͅc̬̟h͡a̫̻̯͘o̫̟̖͍̙̝͉s̗̦̲.̨̹͈̣" + + }; + + IEnumerable documents = specialStrings.Select((specialString) => $@"{{ ""field"" : ""{specialString}""}}"); + await this.CreateIngestQueryDelete( + ConnectionModes.Direct | ConnectionModes.Gateway, + documents, + this.TestOrderByNonAsciiCharactersHelper); + } + + private async Task TestOrderByNonAsciiCharactersHelper( + CosmosContainer container, + IEnumerable documents) + { + foreach (int maxDegreeOfParallelism in new int[] { 1, 100 }) + { + foreach (int maxItemCount in new int[] { 10, 100 }) + { + CosmosQueryRequestOptions feedOptions = new CosmosQueryRequestOptions + { + EnableCrossPartitionQuery = true, + MaxBufferedItemCount = 7000, + MaxConcurrency = maxDegreeOfParallelism + }; + + List actualFromQueryWithoutContinutionTokens = await QueryWithContinuationTokens( + container, + "SELECT * FROM c ORDER BY c.field", + maxItemCount, + feedOptions); + + Assert.AreEqual(documents.Count(), actualFromQueryWithoutContinutionTokens.Count); + } + } + } + + [TestMethod] + public async Task TestMixedTypeOrderBy() + { + int numberOfDocuments = 1 << 4; + int numberOfDuplicates = 1 << 2; + + List documents = new List(numberOfDocuments * numberOfDuplicates); + Random random = new Random(1234); + for (int i = 0; i < numberOfDocuments; ++i) + { + MixedTypedDocument mixedTypeDocument = CrossPartitionQueryTestsOnePartition.GenerateMixedTypeDocument(random); + for (int j = 0; j < numberOfDuplicates; j++) + { + documents.Add(JsonConvert.SerializeObject(mixedTypeDocument)); ; + } + } + + // Just have range indexes + Cosmos.IndexingPolicy indexV1Policy = new Cosmos.IndexingPolicy() + { + IncludedPaths = new Collection() + { + new Cosmos.IncludedPath() + { + Path = "/*", + Indexes = new Collection() + { + Cosmos.Index.Range(Cosmos.DataType.String, -1), + Cosmos.Index.Range(Cosmos.DataType.Number, -1), + } + } + } + }; + + // Add a composite index to force an index v2 container to be made. + Cosmos.IndexingPolicy indexV2Policy = new Cosmos.IndexingPolicy() + { + IncludedPaths = new Collection() + { + new Cosmos.IncludedPath() + { + Path = "/*", + } + }, + + CompositeIndexes = new Collection>() + { + // Simple + new Collection() + { + new Cosmos.CompositePath() + { + Path = "/_ts", + }, + new Cosmos.CompositePath() + { + Path = "/_etag", + } + } + } + }; + + string indexV2Api = HttpConstants.Versions.v2018_09_17; + string indexV1Api = HttpConstants.Versions.v2017_11_15; + + Func, Task> runWithAllowMixedTypeOrderByFlag = async (allowMixedTypeOrderByTestFlag, orderByTypes, expectedExcpetionHandler) => + { + bool allowMixedTypeOrderByTestFlagOriginalValue = OrderByConsumeComparer.AllowMixedTypeOrderByTestFlag; + string apiVersion = allowMixedTypeOrderByTestFlag ? indexV2Api : indexV1Api; + Cosmos.IndexingPolicy indexingPolicy = allowMixedTypeOrderByTestFlag ? indexV2Policy : indexV1Policy; + try + { + OrderByConsumeComparer.AllowMixedTypeOrderByTestFlag = allowMixedTypeOrderByTestFlag; + await this.RunWithApiVersion( + apiVersion, + async () => + { + await this.CreateIngestQueryDelete>>( + ConnectionModes.Direct, + documents, + this.TestMixedTypeOrderByHelper, + new Tuple>(orderByTypes, expectedExcpetionHandler), + "/id", + indexingPolicy); + }); + } + finally + { + OrderByConsumeComparer.AllowMixedTypeOrderByTestFlag = allowMixedTypeOrderByTestFlagOriginalValue; + } + }; + + bool dontAllowMixedTypes = false; + bool doAllowMixedTypes = true; + + OrderByTypes primitives = OrderByTypes.Bool | OrderByTypes.Null | OrderByTypes.Number | OrderByTypes.String; + OrderByTypes nonPrimitives = OrderByTypes.Array | OrderByTypes.Object; + OrderByTypes all = primitives | nonPrimitives | OrderByTypes.Undefined; + + // Don't allow mixed types but single type order by should still work + await runWithAllowMixedTypeOrderByFlag( + dontAllowMixedTypes, + new OrderByTypes[] + { + OrderByTypes.Array, + OrderByTypes.Bool, + OrderByTypes.Null, + OrderByTypes.Number, + OrderByTypes.Object, + OrderByTypes.String, + OrderByTypes.Undefined, + }, null); + + // If you don't allow mixed types but you run a mixed type query then you should get an exception or the results are just wrong. + await runWithAllowMixedTypeOrderByFlag( + dontAllowMixedTypes, + new OrderByTypes[] + { + all, + primitives, + }, + (exception) => + { + Assert.IsTrue( + // Either we get the weird client exception for having mixed types + exception.Message.Contains("Cannot execute cross partition order-by queries on mix types.") + // Or the results are just messed up since the pages in isolation were not mixed typed. + || exception.GetType() == typeof(AssertFailedException)); + }); + + // Mixed type orderby should work for all scenarios, + // since for now the non primitives are accepted to not be served from the index. + await runWithAllowMixedTypeOrderByFlag( + doAllowMixedTypes, + new OrderByTypes[] + { + OrderByTypes.Array, + OrderByTypes.Bool, + OrderByTypes.Null, + OrderByTypes.Number, + OrderByTypes.Object, + OrderByTypes.String, + OrderByTypes.Undefined, + primitives, + nonPrimitives, + all, + }, null); + } + + private sealed class MixedTypedDocument + { + public object MixedTypeField { get; set; } + } + + private static MixedTypedDocument GenerateMixedTypeDocument(Random random) + { + return new MixedTypedDocument() + { + MixedTypeField = GenerateRandomJsonValue(random), + }; + } + + private static object GenerateRandomJsonValue(Random random) + { + switch (random.Next(0, 6)) + { + // Number + case 0: + return random.Next(); + // String + case 1: + return new string('a', random.Next(0, 100)); + // Null + case 2: + return null; + // Bool + case 3: + return (random.Next() % 2) == 0; + // Object + case 4: + return new object(); + // Array + case 5: + return new List(); + default: + throw new ArgumentException(); + } + } + + private sealed class MockOrderByComparer : IComparer + { + public static readonly MockOrderByComparer Value = new MockOrderByComparer(); + + public int Compare(object x, object y) + { + CosmosElement element1 = ObjectToCosmosElement(x); + CosmosElement element2 = ObjectToCosmosElement(y); + + return ItemComparer.Instance.Compare(element1, element2); + } + + private static CosmosElement ObjectToCosmosElement(object obj) + { + string json = JsonConvert.SerializeObject(obj != null ? JToken.FromObject(obj) : JValue.CreateNull()); + byte[] bytes = Encoding.UTF8.GetBytes(json); + return CosmosElement.Create(bytes); + } + } + + [Flags] + private enum OrderByTypes + { + Number = 1 << 0, + String = 1 << 1, + Null = 1 << 2, + Bool = 1 << 3, + Object = 1 << 4, + Array = 1 << 5, + Undefined = 1 << 6, + }; + + private async Task TestMixedTypeOrderByHelper( + CosmosContainer container, + IEnumerable documents, + Tuple> args) + { + OrderByTypes[] orderByTypesList = args.Item1; + Action expectedExceptionHandler = args.Item2; + try + { + foreach (bool isDesc in new bool[] { true, false }) + { + foreach (OrderByTypes orderByTypes in orderByTypesList) + { + string orderString = isDesc ? "DESC" : "ASC"; + List mixedTypeFilters = new List(); + if (orderByTypes.HasFlag(OrderByTypes.Array)) + { + mixedTypeFilters.Add($"IS_ARRAY(c.{nameof(MixedTypedDocument.MixedTypeField)})"); + } + + if (orderByTypes.HasFlag(OrderByTypes.Bool)) + { + mixedTypeFilters.Add($"IS_BOOL(c.{nameof(MixedTypedDocument.MixedTypeField)})"); + } + + if (orderByTypes.HasFlag(OrderByTypes.Null)) + { + mixedTypeFilters.Add($"IS_NULL(c.{nameof(MixedTypedDocument.MixedTypeField)})"); + } + + if (orderByTypes.HasFlag(OrderByTypes.Number)) + { + mixedTypeFilters.Add($"IS_NUMBER(c.{nameof(MixedTypedDocument.MixedTypeField)})"); + } + + if (orderByTypes.HasFlag(OrderByTypes.Object)) + { + mixedTypeFilters.Add($"IS_OBJECT(c.{nameof(MixedTypedDocument.MixedTypeField)})"); + } + + if (orderByTypes.HasFlag(OrderByTypes.String)) + { + mixedTypeFilters.Add($"IS_STRING(c.{nameof(MixedTypedDocument.MixedTypeField)})"); + } + + if (orderByTypes.HasFlag(OrderByTypes.Undefined)) + { + mixedTypeFilters.Add($"not IS_DEFINED(c.{nameof(MixedTypedDocument.MixedTypeField)})"); + } + + string filter = mixedTypeFilters.Count() == 0 ? "true" : string.Join(" OR ", mixedTypeFilters); + + string query = $@" + SELECT VALUE c.{nameof(MixedTypedDocument.MixedTypeField)} + FROM c + WHERE {filter} + ORDER BY c.{nameof(MixedTypedDocument.MixedTypeField)} {orderString}"; + + CosmosQueryRequestOptions feedOptions = new CosmosQueryRequestOptions() + { + MaxBufferedItemCount = 1000, + }; + + List actualFromQueryWithoutContinutionTokens; + actualFromQueryWithoutContinutionTokens = await this.RunQuery( + container, + query, + maxItemCount: 16, + maxConcurrency: 10, + requestOptions: feedOptions); +#if false + For now we can not serve the query through continuation tokens correctly. + This is because we allow order by on mixed types but not comparisions across types + For example suppose the following query: + SELECT c.MixedTypeField FROM c ORDER BY c.MixedTypeField + returns: + [ + {"MixedTypeField":null}, + {"MixedTypeField":false}, + {"MixedTypeField":true}, + {"MixedTypeField":303093052}, + {"MixedTypeField":438985130}, + {"MixedTypeField":"aaaaaaaaaaa"} + ] + and we left off on 303093052 then at some point the cross partition code resumes the query by running the following: + SELECT c.MixedTypeField FROM c WHERE c.MixedTypeField > 303093052 ORDER BY c.MixedTypeField + which will only return the following: + { "MixedTypeField":438985130} + and that is because comparision across types is undefined so "aaaaaaaaaaa" > 303093052 never got emitted +#endif + + IEnumerable insertedDocs = documents + .Select(document => document.GetPropertyValue(nameof(MixedTypedDocument.MixedTypeField))); + + // Build the expected results using LINQ + IEnumerable expected = new List(); + + // Filter based on the mixedOrderByType enum + if (orderByTypes.HasFlag(OrderByTypes.Array)) + { + // no arrays should be served from the range index + } + + if (orderByTypes.HasFlag(OrderByTypes.Bool)) + { + expected = expected.Concat(insertedDocs.Where(x => x is bool)); + } + + if (orderByTypes.HasFlag(OrderByTypes.Null)) + { + expected = expected.Concat(insertedDocs.Where(x => x == null)); + } + + if (orderByTypes.HasFlag(OrderByTypes.Number)) + { + expected = expected.Concat(insertedDocs.Where(x => x is double || x is int || x is long)); + } + + if (orderByTypes.HasFlag(OrderByTypes.Object)) + { + // no objects should be served from the range index + } + + if (orderByTypes.HasFlag(OrderByTypes.String)) + { + expected = expected.Concat(insertedDocs.Where(x => x is string)); + } + + if (orderByTypes.HasFlag(OrderByTypes.Undefined)) + { + // no undefined should be served from the range index + } + + // Order using the mock order by comparer + if (isDesc) + { + expected = expected.OrderByDescending(x => x, MockOrderByComparer.Value); + } + else + { + expected = expected.OrderBy(x => x, MockOrderByComparer.Value); + } + + // bind all the value to JTokens so they can be compared agaisnt the actual. + List expectedBinded = expected.Select(x => x == null ? JValue.CreateNull() : JToken.FromObject(x)).ToList(); + + Assert.IsTrue( + expectedBinded.SequenceEqual(actualFromQueryWithoutContinutionTokens, JsonTokenEqualityComparer.Value), + $@" queryWithoutContinuations: {query}, + expected:{JsonConvert.SerializeObject(expected)}, + actual: {JsonConvert.SerializeObject(actualFromQueryWithoutContinutionTokens)}"); + + // Can't assert for reasons mentioned above + //Assert.IsTrue( + // expected.SequenceEqual(actualFromQueryWithContinutionTokens, DistinctMapTests.JsonTokenEqualityComparer.Value), + // $@" queryWithContinuations: {query}, + // expected:{JsonConvert.SerializeObject(expected)}, + // actual: {JsonConvert.SerializeObject(actualFromQueryWithContinutionTokens)}"); + } + } + } + catch (Exception ex) + { + if (expectedExceptionHandler != null) + { + expectedExceptionHandler(ex); + } + else + { + throw; + } + } + } + + [TestMethod] + public async Task TestQueryCrossPartitionTopOrderBy() + { + int seed = (int)(DateTime.UtcNow - new DateTime(1970, 1, 1)).TotalSeconds; + uint numberOfDocuments = 1000; + string partitionKey = "field_0"; + + QueryOracle.QueryOracleUtil util = new QueryOracle.QueryOracle2(seed); + IEnumerable documents = util.GetDocuments(numberOfDocuments); + + await this.CreateIngestQueryDelete( + ConnectionModes.Direct | ConnectionModes.Gateway, + documents, + this.TestQueryCrossPartitionTopOrderByHelper, + partitionKey, + "/" + partitionKey); + } + + private async Task TestQueryCrossPartitionTopOrderByHelper(CosmosContainer container, IEnumerable documents, string testArg) + { + string partitionKey = testArg; + IDictionary idToRangeMinKeyMap = new Dictionary(); + IRoutingMapProvider routingMapProvider = await this.Client.DocumentClient.GetPartitionKeyRangeCacheAsync(); + + CosmosContainerSettings containerSettings = await container.ReadAsync(); + foreach (Document document in documents) + { + IReadOnlyList targetRanges = await routingMapProvider.TryGetOverlappingRangesAsync( + containerSettings.ResourceId, + Range.GetPointRange( + PartitionKeyInternal.FromObjectArray( + new object[] + { + document.GetValue(partitionKey) + }, + true).GetEffectivePartitionKeyString(containerSettings.PartitionKey))); + Debug.Assert(targetRanges.Count == 1); + idToRangeMinKeyMap.Add(document.Id, targetRanges[0].MinInclusive); + } + + IList partitionKeyValues = new HashSet(documents.Select(doc => doc.GetValue(partitionKey))).ToList(); + + // Test Empty Results + List expectedResults = new List { }; + List computedResults = new List(); + + string emptyQueryText = @"SELECT TOP 5 * FROM Root r WHERE r.partitionKey = 9991123 OR r.partitionKey = 9991124 OR r.partitionKey = 99991125"; + FeedOptions feedOptionsEmptyResult = new FeedOptions + { + EnableCrossPartitionQuery = true + }; + + List queryEmptyResult = await this.RunQuery( + container, + emptyQueryText, + maxConcurrency: 1); + + computedResults = queryEmptyResult.Select(doc => doc.Id).ToList(); + computedResults.Sort(); + expectedResults.Sort(); + + Random rand = new Random(); + Assert.AreEqual(string.Join(",", expectedResults), string.Join(",", computedResults)); + List tasks = new List(); + for (int trial = 0; trial < 1; ++trial) + { + foreach (bool fanOut in new[] { true, false }) + { + foreach (bool isParametrized in new[] { true, false }) + { + foreach (bool hasTop in new[] { false, true }) + { + foreach (bool hasOrderBy in new[] { false, true }) + { + foreach (string sortOrder in new[] { string.Empty, "ASC", "DESC" }) + { + #region Expected Documents + string topValueName = "@topValue"; + int top = rand.Next(4) * rand.Next(partitionKeyValues.Count); + string queryText; + string orderByField = "field_" + rand.Next(10); + IEnumerable filteredDocuments; + + Func getTop = () => + hasTop ? string.Format(CultureInfo.InvariantCulture, "TOP {0} ", isParametrized ? topValueName : top.ToString()) : string.Empty; + + Func getOrderBy = () => + hasOrderBy ? string.Format(CultureInfo.InvariantCulture, " ORDER BY r.{0} {1}", orderByField, sortOrder) : string.Empty; + + if (fanOut) + { + queryText = string.Format( + CultureInfo.InvariantCulture, + "SELECT {0}r.id, r.{1} FROM r{2}", + getTop(), + partitionKey, + getOrderBy()); + + filteredDocuments = documents; + } + else + { + HashSet selectedPartitionKeyValues = new HashSet(partitionKeyValues + .OrderBy(x => rand.Next()) + .ThenBy(x => x) + .Take(rand.Next(1, Math.Min(100, partitionKeyValues.Count) + 1))); + + queryText = string.Format( + CultureInfo.InvariantCulture, + "SELECT {0}r.id, r.{1} FROM r WHERE r.{2} IN ({3}){4}", + getTop(), + partitionKey, + partitionKey, + string.Join(", ", selectedPartitionKeyValues), + getOrderBy()); + + filteredDocuments = documents + .AsParallel() + .Where(doc => selectedPartitionKeyValues.Contains(doc.GetValue(partitionKey))); + } + + if (hasOrderBy) + { + switch (sortOrder) + { + case "": + case "ASC": + filteredDocuments = filteredDocuments + .AsParallel() + .OrderBy(doc => doc.GetValue(orderByField)) + .ThenBy(doc => idToRangeMinKeyMap[doc.Id]) + .ThenBy(doc => int.Parse(doc.Id, CultureInfo.InvariantCulture)); + break; + case "DESC": + filteredDocuments = filteredDocuments + .AsParallel() + .OrderByDescending(doc => doc.GetValue(orderByField)) + .ThenBy(doc => idToRangeMinKeyMap[doc.Id]) + .ThenByDescending(doc => int.Parse(doc.Id, CultureInfo.InvariantCulture)); + break; + } + } + else + { + filteredDocuments = filteredDocuments + .AsParallel() + .OrderBy(doc => idToRangeMinKeyMap[doc.Id]) + .ThenBy(doc => int.Parse(doc.Id, CultureInfo.InvariantCulture)); + } + + if (hasTop) + { + filteredDocuments = filteredDocuments.Take(top); + } + #endregion + #region Actual Documents + IEnumerable actualDocuments; + + int maxDegreeOfParallelism = hasTop ? rand.Next(4) : (rand.Next(2) == 0 ? -1 : (1 + rand.Next(0, 10))); + int? maxItemCount = rand.Next(2) == 0 ? -1 : rand.Next(1, documents.Count()); + CosmosQueryRequestOptions feedOptions = new CosmosQueryRequestOptions + { + MaxBufferedItemCount = rand.Next(2) == 0 ? -1 : rand.Next(Math.Min(100, documents.Count()), documents.Count() + 1), + }; + + if (rand.Next(3) == 0) + { + maxItemCount = null; + } + + CosmosSqlQueryDefinition querySpec = new CosmosSqlQueryDefinition(queryText); + SqlParameterCollection parameters = new SqlParameterCollection(); + if (isParametrized) + { + if (hasTop) + { + querySpec.UseParameter(topValueName, top); + } + } + + DateTime startTime = DateTime.Now; + List result = new List(); + CosmosResultSetIterator query = container.Items.CreateItemQuery( + querySpec, + maxConcurrency: maxDegreeOfParallelism, + requestOptions: feedOptions); + + while (query.HasMoreResults) + { + CosmosQueryResponse response = await query.FetchNextSetAsync(); + result.AddRange(response); + } + + actualDocuments = result; + + #endregion + + double time = (DateTime.Now - startTime).TotalMilliseconds; + + Trace.TraceInformation(": {0}, : {1}, : {2}, : {3}, : {4},