diff --git a/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/CrossPartition/OrderBy/OrderByQueryPartitionRangePageAsyncEnumerator.cs b/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/CrossPartition/OrderBy/OrderByQueryPartitionRangePageAsyncEnumerator.cs index 9d49d9a78e..404dccb513 100644 --- a/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/CrossPartition/OrderBy/OrderByQueryPartitionRangePageAsyncEnumerator.cs +++ b/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/CrossPartition/OrderBy/OrderByQueryPartitionRangePageAsyncEnumerator.cs @@ -158,7 +158,7 @@ public InnerEnumerator CloneWithMaxPageSize() protected override async Task> GetNextPageAsync(ITrace trace, CancellationToken cancellationToken) { - FeedRangeInternal feedRange = HierarchicalPartitionUtils.LimitFeedRangeToSinglePartition(this.PartitionKey, this.FeedRangeState.FeedRange, this.containerQueryProperties); + FeedRangeInternal feedRange = QueryRangeUtils.LimitHpkFeedRangeToPartition(this.PartitionKey, this.FeedRangeState.FeedRange, this.containerQueryProperties); TryCatch monadicQueryPage = await this.queryDataSource .MonadicQueryAsync( diff --git a/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/CrossPartition/Parallel/ParallelCrossPartitionQueryPipelineStage.cs b/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/CrossPartition/Parallel/ParallelCrossPartitionQueryPipelineStage.cs index e5c2cf6b8b..5a2fc378d2 100644 --- a/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/CrossPartition/Parallel/ParallelCrossPartitionQueryPipelineStage.cs +++ b/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/CrossPartition/Parallel/ParallelCrossPartitionQueryPipelineStage.cs @@ -146,7 +146,7 @@ public static TryCatch MonadicCreate( if (targetRanges.Count == 0) { - throw new ArgumentException($"{nameof(targetRanges)} must have some elements"); + return TryCatch.FromResult(new EmptyQueryPipelineStage()); } TryCatch> monadicExtractState = MonadicExtractState(continuationToken, targetRanges); diff --git a/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/CrossPartition/Parallel/QueryPartitionRangePageAsyncEnumerator.cs b/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/CrossPartition/Parallel/QueryPartitionRangePageAsyncEnumerator.cs index ebb57d0781..8e918d16f7 100644 --- a/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/CrossPartition/Parallel/QueryPartitionRangePageAsyncEnumerator.cs +++ b/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/CrossPartition/Parallel/QueryPartitionRangePageAsyncEnumerator.cs @@ -46,7 +46,7 @@ protected override Task> GetNextPageAsync(ITrace trace, Canc throw new ArgumentNullException(nameof(trace)); } - FeedRangeInternal feedRange = HierarchicalPartitionUtils.LimitFeedRangeToSinglePartition(this.partitionKey, this.FeedRangeState.FeedRange, this.containerQueryProperties); + FeedRangeInternal feedRange = QueryRangeUtils.LimitHpkFeedRangeToPartition(this.partitionKey, this.FeedRangeState.FeedRange, this.containerQueryProperties); return this.queryDataSource.MonadicQueryAsync( sqlQuerySpec: this.sqlQuerySpec, feedRangeState: new FeedRangeState(feedRange, this.FeedRangeState.State), diff --git a/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/PipelineFactory.cs b/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/PipelineFactory.cs index b63da96b67..a1603e26f7 100644 --- a/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/PipelineFactory.cs +++ b/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/PipelineFactory.cs @@ -59,7 +59,7 @@ public static TryCatch MonadicCreate( if (targetRanges.Count == 0) { - throw new ArgumentException($"{nameof(targetRanges)} must not be empty."); + return TryCatch.FromResult(new EmptyQueryPipelineStage()); } if (queryInfo == null && hybridSearchQueryInfo == null) diff --git a/Microsoft.Azure.Cosmos/src/Query/Core/QueryClient/CosmosQueryClient.cs b/Microsoft.Azure.Cosmos/src/Query/Core/QueryClient/CosmosQueryClient.cs index e63ca863d3..f872f8dcf3 100644 --- a/Microsoft.Azure.Cosmos/src/Query/Core/QueryClient/CosmosQueryClient.cs +++ b/Microsoft.Azure.Cosmos/src/Query/Core/QueryClient/CosmosQueryClient.cs @@ -23,6 +23,13 @@ public abstract Task GetCachedContainerQueryProperties PartitionKey? partitionKey, ITrace trace, CancellationToken cancellationToken); + + // ISSUE-TODO-adityasa-2025/12/29 - Reduce Coupling: we should not use PartitionKeyRange as return type for this internal interface. + // PartitionKeyRange contains a lot more information (for e.g. RidPrefix, Throughput related information, LSN, parent range id etc), + // none of which is required by callers of these methods. The only information required is min & max values. + // Furthermore, the range is always min-inclusive and max-exclusive (since original PartitionKeyRange is such). + // Callers ultimately convert the returned PartitionKeyRange into a FeedRangeEpk. + // Applies to other methods below as well. /// /// Returns list of effective partition key ranges for a collection. @@ -80,7 +87,7 @@ public abstract Task ExecuteQueryPlanRequestAsync public abstract void ClearSessionTokenCache(string collectionFullName); public abstract Task> GetTargetPartitionKeyRangeByFeedRangeAsync( - string resourceLink, + string resourceLink, string collectionResourceId, Documents.PartitionKeyDefinition partitionKeyDefinition, FeedRangeInternal feedRangeInternal, diff --git a/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/CrossPartition/HierarchicalPartitionUtils.cs b/Microsoft.Azure.Cosmos/src/Query/Core/QueryRangeUtils.cs similarity index 57% rename from Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/CrossPartition/HierarchicalPartitionUtils.cs rename to Microsoft.Azure.Cosmos/src/Query/Core/QueryRangeUtils.cs index 34771387fe..0f2fc818bc 100644 --- a/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/CrossPartition/HierarchicalPartitionUtils.cs +++ b/Microsoft.Azure.Cosmos/src/Query/Core/QueryRangeUtils.cs @@ -2,16 +2,18 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // ------------------------------------------------------------ -namespace Microsoft.Azure.Cosmos.Query.Core.Pipeline.CrossPartition +namespace Microsoft.Azure.Cosmos.Query.Core { using System; using System.Collections.Generic; + using System.Diagnostics; using Microsoft.Azure.Cosmos.Query.Core.QueryClient; using Microsoft.Azure.Documents.Routing; - internal static class HierarchicalPartitionUtils + internal static class QueryRangeUtils { private static readonly bool IsLengthAwareComparisonEnabled = ConfigurationManager.IsLengthAwareRangeComparatorEnabled(); + /// /// Updates the FeedRange to limit the scope of incoming feedRange to logical partition within a single physical partition. /// Generally speaking, a subpartitioned container can experience split partition at any level of hierarchical partition key. @@ -20,7 +22,7 @@ internal static class HierarchicalPartitionUtils /// Since such an epk range does not exist at the container level, Service generates a GoneException. /// This method restrics the range of each enumerator by intersecting it with physical partition range. /// - public static FeedRangeInternal LimitFeedRangeToSinglePartition(PartitionKey? partitionKey, FeedRangeInternal feedRange, ContainerQueryProperties containerQueryProperties) + public static FeedRangeInternal LimitHpkFeedRangeToPartition(PartitionKey? partitionKey, FeedRangeInternal feedRange, ContainerQueryProperties containerQueryProperties) { // We sadly need to check the partition key, since a user can set a partition key in the request options with a different continuation token. // In the future the partition filtering and continuation information needs to be a tightly bounded contract (like cross feed range state). @@ -108,5 +110,88 @@ public static FeedRangeInternal LimitFeedRangeToSinglePartition(PartitionKey? pa return feedRange; } + + /// + /// Limits the partition key ranges to fit within the provided EPK ranges. + /// Computes the overall min and max from the provided ranges, then trims each partition key range to fit within those boundaries. + /// + /// The list of partition key ranges to trim + /// The EPK ranges to use as boundaries + /// A list of trimmed partition key ranges that fit within the provided ranges + public static List LimitPartitionKeyRangesToProvidedRanges( + List partitionKeyRanges, + IReadOnlyList> providedRanges) + { + IComparer> minComparer = IsLengthAwareComparisonEnabled + ? Documents.Routing.Range.LengthAwareMinComparer.Instance + : Documents.Routing.Range.MinComparer.Instance; + + IComparer> maxComparer = IsLengthAwareComparisonEnabled + ? Documents.Routing.Range.LengthAwareMaxComparer.Instance + : Documents.Routing.Range.MaxComparer.Instance; + + // Compute the overall min and max from providedRanges + string overallMin = providedRanges[0].Min; + string overallMax = providedRanges[0].Max; + + foreach (Range providedRange in providedRanges) + { + // ProvidedRanges are user input, which can be generally deserialized from a json representation of FeedRangeInternal. + // FeedRangeInternal allows min/max to be included or excluded. + // However PartitionKeyRange assumes min is inclusive and max is exclusive. + // This is also similar to backend behavior where EPK ranges are always min-inclusive and max-exclusive. + // Therefore, despite the possible customization at FeedRangeInternal level, we only support min-inclusive and max-exclusive ranges. + // Ideally this validation should be done at the public API. Since that is not present, we only assert below. + Debug.Assert(providedRange.IsMinInclusive, "QueryRangeUtils Assert!", "Only min-inclusive ranges are supported!"); + Debug.Assert(!providedRange.IsMaxInclusive, "QueryRangeUtils Assert!", "Only max-exclusive ranges are supported!"); + + if (minComparer.Compare(providedRange, CreateSingleValueRange(overallMin)) < 0) + { + overallMin = providedRange.Min; + } + + if (maxComparer.Compare(providedRange, CreateSingleValueRange(overallMax)) > 0) + { + overallMax = providedRange.Max; + } + } + + // Trim each range to fit within the overall boundaries + List trimmedRanges = new List(partitionKeyRanges.Count); + foreach (Documents.PartitionKeyRange range in partitionKeyRanges) + { + string trimmedMin = range.MinInclusive; + string trimmedMax = range.MaxExclusive; + + // Trim min: use the greater of range.Min and overallMin + if (minComparer.Compare(CreateSingleValueRange(range.MinInclusive), CreateSingleValueRange(overallMin)) < 0) + { + trimmedMin = overallMin; + } + + // Trim max: use the lesser of range.Max and overallMax + if (maxComparer.Compare(CreateSingleValueRange(range.MaxExclusive), CreateSingleValueRange(overallMax)) > 0) + { + trimmedMax = overallMax; + } + + trimmedRanges.Add( + new Documents.PartitionKeyRange + { + Id = range.Id, + MinInclusive = trimmedMin, + MaxExclusive = trimmedMax, + Parents = range.Parents + }); + } + + return trimmedRanges; + } + + private static Range CreateSingleValueRange(string singleValue) => new Range( + singleValue, + singleValue, + isMinInclusive: true, + isMaxInclusive: true); } } diff --git a/Microsoft.Azure.Cosmos/src/Query/v3Query/CosmosQueryClientCore.cs b/Microsoft.Azure.Cosmos/src/Query/v3Query/CosmosQueryClientCore.cs index 69803e14f2..2f5b9f15c5 100644 --- a/Microsoft.Azure.Cosmos/src/Query/v3Query/CosmosQueryClientCore.cs +++ b/Microsoft.Azure.Cosmos/src/Query/v3Query/CosmosQueryClientCore.cs @@ -231,14 +231,16 @@ public override async Task> GetTargetPartitionKeyRangeBy using (ITrace childTrace = trace.StartChild("Get Overlapping Feed Ranges", TraceComponent.Routing, Tracing.TraceLevel.Info)) { IRoutingMapProvider routingMapProvider = await this.GetRoutingMapProviderAsync(); - List> ranges = await feedRangeInternal.GetEffectiveRangesAsync(routingMapProvider, collectionResourceId, partitionKeyDefinition, trace); + List> providedRanges = await feedRangeInternal.GetEffectiveRangesAsync(routingMapProvider, collectionResourceId, partitionKeyDefinition, trace); - return await this.GetTargetPartitionKeyRangesAsync( + List ranges = await this.GetTargetPartitionKeyRangesAsync( resourceLink, collectionResourceId, - ranges, + providedRanges, forceRefresh, childTrace); + + return QueryRangeUtils.LimitPartitionKeyRangesToProvidedRanges(ranges, providedRanges); } } @@ -280,8 +282,8 @@ public override async Task> GetTargetPartitionKeyRangesA if (ranges == null) { throw new NotFoundException($"{DateTime.UtcNow.ToString("o", CultureInfo.InvariantCulture)}: GetTargetPartitionKeyRanges(collectionResourceId:{collectionResourceId}, providedRanges: {string.Join(",", providedRanges)} failed due to stale cache"); - } - + } + return ranges; } } diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/QueryFeedRangeTest.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/QueryFeedRangeTest.cs new file mode 100644 index 0000000000..0d0da25e31 --- /dev/null +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/QueryFeedRangeTest.cs @@ -0,0 +1,506 @@ +namespace Microsoft.Azure.Cosmos.EmulatorTests.Query +{ + using System; + using System.Collections.Generic; + using System.Collections.ObjectModel; + using System.Linq; + using System.Threading.Tasks; + using Microsoft.Azure.Cosmos; + using Microsoft.Azure.Cosmos.CosmosElements; + using Microsoft.Azure.Documents; + using Microsoft.VisualStudio.TestTools.UnitTesting; + + [TestClass] + public class QueryFeedRangeTest : QueryTestsBase + { + private static readonly List SampleDocuments = new List + { + @"{""id"":""id0"",""operation"":""create"",""duration"":45,""tenant"":""tenant0"",""user"":""user0"",""session"":""session0""}", + @"{""id"":""id1"",""operation"":""update"",""duration"":23,""tenant"":""tenant0"",""user"":""user0"",""session"":""session1""}", + @"{""id"":""id2"",""operation"":""delete"",""duration"":67,""tenant"":""tenant0"",""user"":""user1"",""session"":""session0""}", + @"{""id"":""id3"",""operation"":""create"",""duration"":89,""tenant"":""tenant0"",""user"":""user1"",""session"":""session1""}", + @"{""id"":""id4"",""operation"":""update"",""duration"":12,""tenant"":""tenant1"",""user"":""user0"",""session"":""session0""}", + @"{""id"":""id5"",""operation"":""delete"",""duration"":56,""tenant"":""tenant1"",""user"":""user0"",""session"":""session1""}", + @"{""id"":""id6"",""operation"":""create"",""duration"":34,""tenant"":""tenant1"",""user"":""user1"",""session"":""session0""}", + @"{""id"":""id7"",""operation"":""update"",""duration"":78,""tenant"":""tenant1"",""user"":""user1"",""session"":""session1""}", + @"{""id"":""id8"",""operation"":""delete"",""duration"":91,""tenant"":""tenant2"",""user"":""user0"",""session"":""session0""}", + @"{""id"":""id9"",""operation"":""create"",""duration"":5,""tenant"":""tenant2"",""user"":""user1"",""session"":""session1""}", + @"{""id"":""id10"",""operation"":""update"",""duration"":42,""tenant"":""tenant0"",""user"":""user0"",""session"":""session2""}", + @"{""id"":""id11"",""operation"":""delete"",""duration"":73,""tenant"":""tenant1"",""user"":""user2"",""session"":""session0""}", + @"{""id"":""id12"",""operation"":""create"",""duration"":28,""tenant"":""tenant2"",""user"":""user0"",""session"":""session1""}", + @"{""id"":""id13"",""operation"":""update"",""duration"":61,""tenant"":""tenant0"",""user"":""user2"",""session"":""session2""}", + @"{""id"":""id14"",""operation"":""delete"",""duration"":15,""tenant"":""tenant1"",""user"":""user1"",""session"":""session2""}", + }; + + private static PartitionKeyDefinition HierarchicalPartitionKeyDefinition => + new PartitionKeyDefinition + { + Paths = new Collection { "/tenant", "/user", "/session" }, + Kind = PartitionKind.MultiHash, + Version = Documents.PartitionKeyDefinitionVersion.V2 + }; + + private static PartitionKeyDefinition SinglePartitionKeyDefinition => + new PartitionKeyDefinition + { + Paths = new Collection { "/tenant" }, + Kind = PartitionKind.Hash, + Version = Documents.PartitionKeyDefinitionVersion.V2 + }; + + [TestMethod] + public async Task TestSinglePartitionKeyContainer_Direct_SinglePartition() + { + await this.ExecuteTest( + SinglePartitionKeyDefinition, + ConnectionModes.Direct, + CollectionTypes.MultiPartition); + } + + [TestMethod] + public async Task TestSinglePartitionKeyContainer_Gateway_SinglePartition() + { + await this.ExecuteTest( + SinglePartitionKeyDefinition, + ConnectionModes.Gateway, + CollectionTypes.SinglePartition); + } + + [TestMethod] + public async Task TestSinglePartitionKeyContainer_Direct_MultiPartition() + { + await this.ExecuteTest( + SinglePartitionKeyDefinition, + ConnectionModes.Direct, + CollectionTypes.MultiPartition); + } + + [TestMethod] + public async Task TestSinglePartitionKeyContainer_Gateway_MultiPartition() + { + await this.ExecuteTest( + SinglePartitionKeyDefinition, + ConnectionModes.Gateway, + CollectionTypes.MultiPartition); + } + + [TestMethod] + public async Task TestHierarchicalPartitionKeyContainer_Direct_SinglePartition() + { + await this.ExecuteTest( + HierarchicalPartitionKeyDefinition, + ConnectionModes.Direct, + CollectionTypes.MultiPartition); + } + + [TestMethod] + public async Task TestHierarchicalPartitionKeyContainer_Gateway_SinglePartition() + { + await this.ExecuteTest( + HierarchicalPartitionKeyDefinition, + ConnectionModes.Gateway, + CollectionTypes.SinglePartition); + } + + [TestMethod] + public async Task TestHierarchicalPartitionKeyContainer_Direct_MultiPartition() + { + await this.ExecuteTest( + HierarchicalPartitionKeyDefinition, + ConnectionModes.Direct, + CollectionTypes.MultiPartition); + } + + [TestMethod] + public async Task TestHierarchicalPartitionKeyContainer_Gateway_MultiPartition() + { + await this.ExecuteTest( + HierarchicalPartitionKeyDefinition, + ConnectionModes.Gateway, + CollectionTypes.MultiPartition); + } + + private async Task ExecuteTest(PartitionKeyDefinition partitionKeyDefinition, ConnectionModes connectionMode, CollectionTypes collectionType) + { + await this.CreateIngestQueryDeleteAsync( + connectionMode, + collectionType, + SampleDocuments, + (container, documents) => this.ExecuteAllQueries(container, partitionKeyDefinition, documents), + partitionKeyDefinition, + IndexingPolicy); + } + + private static Cosmos.IndexingPolicy IndexingPolicy => + new Cosmos.IndexingPolicy + { + Automatic = true, + IndexingMode = Cosmos.IndexingMode.Consistent, + IncludedPaths = new Collection + { + new Cosmos.IncludedPath + { + Path = "/*" + } + } + }; + + private static string GetEpk(PartitionKeyDefinition partitionKeyDefinition, params string[] values) + { + if (partitionKeyDefinition == null) + { + throw new ArgumentNullException(nameof(partitionKeyDefinition)); + } + + if (values == null || values.Length == 0) + { + throw new ArgumentException("Values array must have at least one element", nameof(values)); + } + + if (values.Length > 3) + { + throw new ArgumentException("Values array cannot have more than 3 elements", nameof(values)); + } + + if (values.Length != partitionKeyDefinition.Paths.Count) + { + throw new ArgumentException( + $"Number of values ({values.Length}) must match number of partition key paths ({partitionKeyDefinition.Paths.Count})", + nameof(values)); + } + + // Build the partition key from the values + PartitionKeyBuilder pkBuilder = new PartitionKeyBuilder(); + + foreach (string value in values) + { + if (value == null) + { + pkBuilder.AddNullValue(); + } + else + { + pkBuilder.Add(value); + } + } + + Cosmos.PartitionKey pk = pkBuilder.Build(); + + string epk = pk.InternalKey.GetEffectivePartitionKeyString(partitionKeyDefinition); + + return epk; + } + + private async Task ExecuteAllQueries( + Container container, + PartitionKeyDefinition partitionKeyDefinition, + IReadOnlyList documents) + { + Console.WriteLine("\n=== Executing All Query Tests ===\n"); + + // Queries must project full partition key values from the documents hit by the query. + // The projected pk values are used to validate that only documents within the FeedRange are returned. + List queries = new List + { + @" + SELECT * + FROM c", + + @" + SELECT * + FROM c + WHERE STARTSWITH(c.tenant, 'tenant')", + + @" + SELECT * + FROM c + WHERE c.operation = 'create' OR c.operation = 'delete'", + + @" + SELECT + COUNT(1) count, + c.tenant, + c.session, + c.user + FROM c + GROUP BY c.tenant, c.session, c.user", + + @" + SELECT * + FROM c + ORDER BY c.tenant", + + @" + SELECT + min(c.duration) AS minDuration, + max(c.duration) AS maxDuration, + sum(c.duration) AS sumDuration, + c.tenant, + c.session, + c.user + FROM c + GROUP BY c.tenant, c.session, c.user + ORDER BY c.tenant" + }; + + // Step 1: Get distinct partition key values + Console.WriteLine("Step 1: Extracting distinct partition key values from documents..."); + List distinctPkValues = GetDistinctPartitionKeyValues(documents, partitionKeyDefinition); + Console.WriteLine($"Found {distinctPkValues.Count} distinct partition key value combinations\n"); + + // Step 2: Generate EPKs for each distinct partition key value + Console.WriteLine("Step 2: Generating EPK values..."); + List<(string[] pkValues, string epk)> epkList = GenerateEpksForPartitionKeys(distinctPkValues, partitionKeyDefinition); + Console.WriteLine($"Generated {epkList.Count} EPK values\n"); + + // Step 3: Sort EPKs and create ranges between consecutive values + Console.WriteLine("Step 3: Sorting EPK values and creating ranges..."); + List<(string[] pkValues, string epk)> sortedEpks = epkList.OrderBy(item => item.epk, StringComparer.Ordinal).ToList(); + Console.WriteLine($"Sorted {sortedEpks.Count} unique EPK values\n"); + + // Print sorted EPKs + foreach ((string[] pkValues, string epk) in sortedEpks) + { + Console.WriteLine($" EPK: {epk}, PK Values: [{string.Join(", ", pkValues)}]"); + } + + // Step 4: Create FeedRanges from each minEpk to all subsequent maxEpks and execute queries + for (int i = 0; i < sortedEpks.Count - 1; i++) + { + (string[] minPkValues, string minEpk) = sortedEpks[i]; + + // Nested loop: for each minEpk, test ranges to all subsequent maxEpks + for (int j = i; j < sortedEpks.Count; j++) + { + (string[] maxPkValues, string maxEpk) = sortedEpks[j]; + + Console.WriteLine($"\n====== Executing queries for EPK range [min:{i + 1}, max:{j + 1}] ======"); + Console.WriteLine($"Min EPK: {minEpk} (inclusive)"); + Console.WriteLine($"Max EPK: {maxEpk} (exclusive)"); + + // Print all PK values within the range (inclusive) + Console.WriteLine("PkValues: ["); + for (int pkIndex = i; pkIndex <= j; pkIndex++) + { + (string[] pkValues, string epk) = sortedEpks[pkIndex]; + Console.WriteLine($" [{string.Join(", ", pkValues)}]" + (pkIndex == j ? " (exclusive)" : "(inclusive),")); + } + Console.WriteLine(")"); + + // Create FeedRange from min to max EPK values + FeedRange feedRange = CreateFeedRangeFromEpkRange(minEpk, maxEpk); + Console.WriteLine($"FeedRange: {feedRange.ToJsonString()}\n"); + + for (int k = 0; k < queries.Count; k++) + { + string query = queries[k]; + Console.WriteLine("------------"); + Console.WriteLine($"Query {k + 1}:{query}\n"); + + FeedIterator iterator = container.GetItemQueryIterator( + feedRange, + new QueryDefinition(query)); + List results = new List(); + while (iterator.HasMoreResults) + { + FeedResponse response = await iterator.ReadNextAsync(); + results.AddRange(response); + } + + Console.WriteLine($"Results: {results.Count} documents:"); + foreach (CosmosElement result in results) + { + Console.WriteLine($" {result}"); + } + + // Validate EPK ranges + await this.ValidateResultsWithinFeedRange(results, feedRange, partitionKeyDefinition); + Console.WriteLine("------------\n"); + } + } + } + + Console.WriteLine("\n=== All Queries Completed ===\n"); + } + + private static List GetDistinctPartitionKeyValues( + IReadOnlyList documents, + PartitionKeyDefinition partitionKeyDefinition) + { + HashSet distinctValuesSet = new HashSet(); + List distinctValuesList = new List(); + int pathCount = partitionKeyDefinition.Paths.Count; + + foreach (CosmosObject document in documents) + { + string[] pkValues = ExtractPartitionKeyValues(document, partitionKeyDefinition); + + if (pathCount == 1) + { + // Single partition key: add the single value + string key = string.Join("|", pkValues.Select(v => v ?? "<>")); + if (distinctValuesSet.Add(key)) + { + distinctValuesList.Add(pkValues); + } + } + else + { + // Hierarchical partition key: generate all prefixes (1, 2, and 3 elements) + for (int prefixLength = 1; prefixLength <= pathCount; prefixLength++) + { + string[] prefixValues = pkValues.Take(prefixLength).ToArray(); + string key = string.Join("|", prefixValues.Select(v => v ?? "<>")); + + if (distinctValuesSet.Add(key)) + { + distinctValuesList.Add(prefixValues); + } + } + } + } + + return distinctValuesList; + } + + private static List<(string[] pkValues, string epk)> GenerateEpksForPartitionKeys( + List distinctPkValues, + PartitionKeyDefinition partitionKeyDefinition) + { + List<(string[] pkValues, string epk)> epkList = new List<(string[], string)>(); + + foreach (string[] pkValues in distinctPkValues) + { + // Create a partition key definition matching the number of values + PartitionKeyDefinition pkDef = new PartitionKeyDefinition + { + Paths = new Collection(partitionKeyDefinition.Paths.Take(pkValues.Length).ToList()), + Kind = partitionKeyDefinition.Kind, + Version = partitionKeyDefinition.Version + }; + + string epk = GetEpk(pkDef, pkValues); + epkList.Add((pkValues, epk)); + } + + return epkList; + } + + private static FeedRange CreateFeedRangeFromEpkRange(string minEpk, string maxEpk) + { + string feedRangeSerialization = Newtonsoft.Json.JsonConvert.SerializeObject(new + { + Range = new { min = minEpk, max = maxEpk } + }); + + return FeedRange.FromJsonString(feedRangeSerialization); + } + + private async Task ValidateResultsWithinFeedRange( + List results, + FeedRange feedRange, + PartitionKeyDefinition partitionKeyDefinition) + { + // Get the EPK ranges from the FeedRange + IReadOnlyList> epkRanges = await ((FeedRangeInternal)feedRange).GetEffectiveRangesAsync( + await this.Client.DocumentClient.GetPartitionKeyRangeCacheAsync(Cosmos.Tracing.NoOpTrace.Singleton), + null, + partitionKeyDefinition, + Cosmos.Tracing.NoOpTrace.Singleton); + + int validCount = 0; + int invalidCount = 0; + + foreach (CosmosElement result in results) + { + if (result is CosmosObject cosmosObject) + { + // Extract partition key values from the result + string[] pkValues = ExtractPartitionKeyValues(cosmosObject, partitionKeyDefinition); + + if (pkValues != null) + { + // Get the EPK for this document + string documentEpk = GetEpk(partitionKeyDefinition, pkValues); + + // Check if the EPK is within any of the FeedRange EPK ranges + bool isWithinRange = false; + foreach (Documents.Routing.Range epkRange in epkRanges) + { + if (IsEpkWithinRange(documentEpk, epkRange)) + { + isWithinRange = true; + break; + } + } + + if (isWithinRange) + { + validCount++; + } + else + { + invalidCount++; + Console.WriteLine($" WARNING: Document with EPK '{documentEpk}' is outside FeedRange"); + } + } + } + } + + Console.WriteLine($"\nValidation: {validCount} documents within range, {invalidCount} documents outside range"); + + if (invalidCount > 0) + { + Assert.Fail($"Found {invalidCount} documents outside the specified FeedRange"); + } + } + + private static string[] ExtractPartitionKeyValues(CosmosObject cosmosObject, PartitionKeyDefinition partitionKeyDefinition) + { + string[] values = new string[partitionKeyDefinition.Paths.Count]; + + for (int i = 0; i < partitionKeyDefinition.Paths.Count; i++) + { + string path = partitionKeyDefinition.Paths[i]; + string propertyName = path.TrimStart('/'); + + if (cosmosObject.TryGetValue(propertyName, out CosmosElement element)) + { + if (element is CosmosString cosmosString) + { + values[i] = cosmosString.Value; + } + else if (element is CosmosNull) + { + values[i] = null; + } + else + { + // For other types, convert to string representation + values[i] = element.ToString(); + } + } + else + { + values[i] = null; + } + } + + return values; + } + + private static bool IsEpkWithinRange(string epk, Documents.Routing.Range range) + { + // Check if epk is >= or > range.Min depending on IsMinInclusive + int minComparisonValue = string.Compare(epk, range.Min, StringComparison.Ordinal); + bool isMinWithinRange = range.IsMinInclusive ? (minComparisonValue >= 0) : (minComparisonValue > 0); + + // Check if epk is < or <= range.Max depending on IsMaxInclusive + int maxComparisonValue = string.Compare(epk, range.Max, StringComparison.Ordinal); + bool isMaxWithinRange = range.IsMaxInclusive ? (maxComparisonValue <= 0) : (maxComparisonValue < 0); + + return isMinWithinRange && isMaxWithinRange; + } + } +} + diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/QueryTestsBase.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/QueryTestsBase.cs index c947041b85..827bd2d054 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/QueryTestsBase.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/QueryTestsBase.cs @@ -823,7 +823,7 @@ internal static Task> RunQueryCombinationsAsync( return RunQueryCombinationsAsync(container, query, queryRequestOptions, queryDrainingMode); } - internal static async Task> RunQueryCombinationsAsync( + internal static async Task> RunQueryCombinationsAsync( Container container, string query, QueryRequestOptions queryRequestOptions, diff --git a/changelog.md b/changelog.md index 05494743dd..19cb20ceb0 100644 --- a/changelog.md +++ b/changelog.md @@ -1808,6 +1808,7 @@ Below is a list of any know issues affecting the [recommended minimum version](# | Issue | Impact | Mitigation | Tracking link | | --- | --- | --- | --- | +| Calling `GetItemQueryIterator` with FeedRange | Scenarios that use FeedRange with min excluded or max included while calling `GetItemQueryIterator`. | `FeedIterator` created by calling `GetItemQueryIterator` while supplying FeedRange ignores the max/min inclusion properties. The returned iterator _always_ includes min and excludes max implicitly. This will be tracked and fixed as a separate issue. As a workaround, only use FeedRange values with min included and max excluded. | TBD | | Optimistic Direct Execution in case of Partition Split/Merge. | Scenarios that enable optimistic direct execution. | Optimistic Direct Execution may result in incorrect results when partition split/merge occurs in the backend while query execution is in progress. General recommendation is disable optimistic direct execution while executing queries. | [#4971](https://github.com/Azure/azure-cosmos-dotnet-v3/issues/4971) | | Optimistic Direct execution continuation token. | Scenarios that enable optimistic direct execution. | Optimistic Direct Execution may produce a continuation token that is rejected by SDK after partition split. General recommendation is disable optimistic direct execution while executing queries. | [#4972](https://github.com/Azure/azure-cosmos-dotnet-v3/issues/4972) | | `FeedIterator` enters an infinite loop after a physical partition split occurs in a container using hierarchical partition keys. | Queries using prefix partition keys. | Rather than having the PK included in the query request options, filtering on top level hierarchical Pks should be done through where clauses. **NOTE:** This issue has been fixed in version 3.39.0 | [#4326](https://github.com/Azure/azure-cosmos-dotnet-v3/issues/4326) |