diff --git a/.github/agents/msdata-direct-sync-agent.agent.md b/.github/agents/msdata-direct-sync-agent.agent.md new file mode 100644 index 0000000000..2bffc3cfbb --- /dev/null +++ b/.github/agents/msdata-direct-sync-agent.agent.md @@ -0,0 +1,490 @@ +# Copilot Agent: msdata/direct Branch Sync +## Azure Cosmos DB .NET SDK (azure-cosmos-dotnet-v3) + +--- + +## Quick Start Prompt + +**Copy-paste this prompt to start the sync workflow:** + +``` +Follow the msdata/direct sync agent plan in .github/agents/msdata-direct-sync-agent.agent.md + +Sync the msdata/direct branch with the latest v3 master and msdata direct codebase. +``` + +**What the agent will do:** +1. Verify environment setup (git, .NET SDK, repo clones) +2. Prompt for msdata CosmosDB repo path +3. Create feature branch from `msdata/direct` +4. Merge latest `master` into feature branch +5. Run `msdata_sync.ps1` to sync direct package files +6. **Verify sync completeness** — scan msdata source dirs for files missed by the script and auto-copy them +7. Build and validate +8. Create PR to `msdata/direct` with proper formatting +9. Monitor CI pipeline + +--- + +## 0. Prerequisites + +### 0.1 Required Tools + +```yaml +required_tools: + git: + verify: "git --version" + minimum: "2.x" + + dotnet_sdk: + verify: "dotnet --version" + minimum: "8.0" + + gh_cli: + verify: "gh auth status" + purpose: "PR creation and monitoring" + setup: "gh auth login --web" + + powershell: + verify: "$PSVersionTable.PSVersion" + minimum: "5.1" +``` + +### 0.2 Required Repository Clones + +```yaml +required_repos: + azure_cosmos_dotnet_v3: + url: "https://github.com/Azure/azure-cosmos-dotnet-v3.git" + required_branches: + - master + - msdata/direct + verify: "git branch -a | Select-String 'msdata/direct'" + + msdata_cosmosdb: + description: "Internal CosmosDB repository (msdata)" + note: "User will be prompted for local path at runtime" + required_branch: "master" + verify: "Test-Path " +``` + +### 0.3 Verification Checklist + +```markdown +## Pre-Sync Verification + +- [ ] git installed and configured: `git config user.name` +- [ ] .NET SDK installed: `dotnet --version` +- [ ] gh CLI authenticated: `gh auth status` +- [ ] v3 repo cloned with msdata/direct branch available +- [ ] msdata CosmosDB repo cloned and on master branch +``` + +--- + +## 1. Core Principles + +> ⚠️ **These principles apply to ALL phases of the sync workflow.** + +```yaml +sync_principles: + rules: + - "ALWAYS verify each phase before proceeding to the next" + - "ALWAYS accept incoming master changes when resolving merge conflicts" + - "NEVER force-push to msdata/direct directly" + - "ALWAYS create PRs as draft first" + - "ALWAYS run a clean build before creating the PR" + - "Prompt user for input when paths or decisions are needed" + + evidence_required: + merge_complete: "Git merge output showing success or resolved conflicts" + sync_complete: "msdata_sync.ps1 output showing all files copied" + build_passed: "dotnet build output showing 'Build succeeded. 0 Error(s)'" + pr_created: "PR URL from gh pr create" +``` + +--- + +## 2. Workflow Phases + +### Phase 1: Environment Setup & Validation + +**Goal:** Ensure all prerequisites are met and gather required user input. + +```yaml +phase_1_steps: + step_1_verify_tools: + commands: + - "git --version" + - "dotnet --version" + - "gh auth status" + action: "If any tool is missing, guide user through installation" + + step_2_verify_v3_repo: + commands: + - "git remote -v" + - "git fetch origin --quiet" + - "git branch -a | Select-String 'msdata/direct'" + verify: "origin points to Azure/azure-cosmos-dotnet-v3 and msdata/direct exists" + + step_3_prompt_msdata_path: + action: "Ask user for local path to msdata CosmosDB repo" + question: "What is the local path to your msdata CosmosDB repository clone?" + examples: + - "Q:\\CosmosDB" + - "C:\\repos\\CosmosDB" + - "E:\\src\\CosmosDB" + validation: "Test-Path && Test-Path \\sdk" + + step_4_verify_msdata_repo: + commands: + - "cd && git status" + - "cd && git branch --show-current" + action: "Ensure msdata repo is on master and up to date" + fix: "cd && git checkout master && git pull" +``` + +### Phase 2: Branch Preparation + +**Goal:** Create a feature branch from `msdata/direct` and merge latest `master`. + +```yaml +phase_2_steps: + step_1_update_branches: + commands: + - "git fetch origin master:master --quiet" + - "git fetch origin msdata/direct --quiet" + verify: "Both branches are up to date with remote" + + step_2_create_feature_branch: + description: "Create feature branch from msdata/direct" + naming_convention: "users//update_msdata_direct_" + commands: + - "git checkout msdata/direct" + - 'git checkout -b users//update_msdata_direct_' + example: "git checkout -b users/nalutripician/update_msdata_direct_03_03_2026" + get_username: "git config user.name or gh api user --jq '.login'" + get_date: "(Get-Date).ToString('MM_dd_yyyy')" + + step_3_merge_master: + commands: + - "git merge master" + expect: "Merge conflicts are likely" + + step_4_resolve_conflicts: + strategy: "Accept incoming master changes for most conflicts" + commands: + - "git checkout --theirs " + - "git add " + - "git merge --continue" + manual_review_needed: + - "Files in Microsoft.Azure.Cosmos/src/direct/ — these may need careful review" + - "Project files (.csproj) — ensure both sets of changes are preserved" + - "Directory.Build.props — version numbers need careful handling" + notes: + - "If conflicts are too complex, ask user for guidance" + - "Document all conflict resolutions for PR description" +``` + +### Phase 3: msdata File Sync + +**Goal:** Copy latest Microsoft.Azure.Cosmos.Direct files from msdata repo. + +```yaml +phase_3_steps: + step_1_locate_sync_script: + path: "Microsoft.Azure.Cosmos/src/direct/msdata_sync.ps1" + verify: "Test-Path Microsoft.Azure.Cosmos/src/direct/msdata_sync.ps1" + fallback: "If script doesn't exist after merge, check msdata/direct branch directly" + + step_2_configure_sync_script: + description: "Update msdata_sync.ps1 with user-provided msdata repo path" + action: "Replace the $baseDir value with the user-provided path" + pattern: '$baseDir = "\\CosmosDB"' + replacement: '$baseDir = ""' + important: "Do NOT commit this path change — revert after sync" + + step_3_run_sync_script: + commands: + - "cd Microsoft.Azure.Cosmos/src/direct" + - ".\\msdata_sync.ps1" + expect: "Console output showing files being copied" + success_indicator: "Script completes without Write-Error lines" + + step_4_verify_and_copy_missing_files: + description: > + IMPORTANT: msdata_sync.ps1 only copies files that already exist locally. + New files added in the msdata repo will be silently missed. This step + performs a reverse scan of all msdata source directories and auto-copies + any .cs files that are not yet present in the v3 direct/ folder. + msdata_source_directories: + - "\\Product\\SDK\\.net\\Microsoft.Azure.Cosmos.Direct\\src\\" + - "\\Product\\Microsoft.Azure.Documents\\Common\\SharedFiles\\" + - "\\Product\\Microsoft.Azure.Documents\\SharedFiles\\Routing\\" + - "\\Product\\Microsoft.Azure.Documents\\SharedFiles\\Rntbd2\\" + - "\\Product\\Microsoft.Azure.Documents\\SharedFiles\\Rntbd\\" + - "\\Product\\Microsoft.Azure.Documents\\SharedFiles\\Rntbd\\rntbdtokens\\" + - "\\Product\\SDK\\.net\\Microsoft.Azure.Documents.Client\\LegacyXPlatform\\" + - "\\Product\\Cosmos\\Core\\Core.Trace\\" + - "\\Product\\Cosmos\\Core\\Core\\Utilities\\" + - "\\Product\\Microsoft.Azure.Documents\\SharedFiles\\" + - "\\Product\\Microsoft.Azure.Documents\\SharedFiles\\Collections\\" + - "\\Product\\Microsoft.Azure.Documents\\SharedFiles\\Query\\" + - "\\Product\\Microsoft.Azure.Documents\\SharedFiles\\Management\\" + exclude_files: + - "AssemblyKeys.cs" + - "BaseTransportClient.cs" + - "CpuReaderBase.cs" + - "LinuxCpuReader.cs" + - "MemoryLoad.cs" + - "MemoryLoadHistory.cs" + - "UnsupportedCpuReader.cs" + - "WindowsCpuReader.cs" + - "msdata_sync.ps1" + procedure: + - step: "For each msdata source directory, list all .cs files" + command: 'Get-ChildItem "\\" -Filter "*.cs" -File -ErrorAction SilentlyContinue' + - step: "For each .cs file found, check if it already exists in Microsoft.Azure.Cosmos/src/direct/" + note: "Files from the Rntbd2 source dir go into the direct/rntbd2/ subdirectory" + - step: "If the file does NOT exist locally and is NOT in the exclude list, copy it" + command: 'Copy-Item "\\\\" -Destination "Microsoft.Azure.Cosmos/src/direct/" -Force' + - step: "Log every file that was auto-copied so it can be included in the PR description" + - step: "After all directories are scanned, report a summary" + success_criteria: "No new files remain uncopied from any msdata source directory" + notes: + - "The Rntbd2 directory is special — its files go to direct/rntbd2/, not direct/" + - "TransportClient.cs, RMResources.Designer.cs, and RMResources.resx are handled separately by the sync script and should be skipped in this check" + - "If any files are copied, re-run msdata_sync.ps1 afterward to ensure consistency" + - "If using the helper script, this verification runs automatically as part of the Sync phase" + + step_5_revert_script_path: + description: "Revert the $baseDir change in msdata_sync.ps1" + commands: + - "git checkout -- Microsoft.Azure.Cosmos/src/direct/msdata_sync.ps1" + verify: "git diff Microsoft.Azure.Cosmos/src/direct/msdata_sync.ps1 shows no changes" +``` + +### Phase 4: Build Validation + +**Goal:** Verify the merged and synced code builds successfully. + +```yaml +phase_4_steps: + step_1_clean_build: + command: "dotnet build Microsoft.Azure.Cosmos.sln -c Release" + expected: "Build succeeded. 0 Error(s)" + timeout: "5-10 minutes" + + step_2_fix_build_errors: + description: "If build fails, investigate and fix errors" + common_issues: + missing_files: + symptom: "CS file not found or type not defined" + fix: "Check if file was missed during sync, copy from msdata repo" + namespace_conflicts: + symptom: "Ambiguous reference or namespace conflict" + fix: "Check using statements, resolve with fully qualified names" + api_changes: + symptom: "Method signature mismatch" + fix: "Update to match latest API from master or direct" + action: "Fix errors, rebuild, repeat until clean" + + step_3_verify_build: + command: "dotnet build Microsoft.Azure.Cosmos.sln -c Release" + required: "Build MUST succeed before proceeding" + evidence: "Capture build output as proof" +``` + +### Phase 5: PR Creation & Submission + +**Goal:** Push the feature branch and create a properly formatted PR. + +```yaml +phase_5_steps: + step_1_stage_and_commit: + commands: + - "git add -A" + - 'git commit -m "[Internal] Direct package: Adds msdata/direct update from master"' + verify: "git status shows clean working tree" + + step_2_push_branch: + command: "git push origin users//update_msdata_direct_" + verify: "Push succeeds without errors" + + step_3_create_pr: + command: | + gh pr create --draft \ + --base msdata/direct \ + --title "[Internal] Direct package: Adds msdata/direct update from master" \ + --body "" + pr_description_template: | + # Pull Request Template + + ## Description + + Syncs the `msdata/direct` branch with: + - Latest `master` branch (v3 SDK changes) + - Latest `Microsoft.Azure.Cosmos.Direct` files from msdata CosmosDB repo + + ### Changes Include + - Merged latest `master` branch into `msdata/direct` + - Updated `Microsoft.Azure.Cosmos.Direct` files via `msdata_sync.ps1` + - Resolved merge conflicts (accepted master changes) + - Build validated: `dotnet build` passes + + ## Type of change + + - [x] New feature (non-breaking change which adds functionality) + + ## Validation + + - [x] Local build passes (`dotnet build Microsoft.Azure.Cosmos.sln -c Release`) + + reviewers: + - "kirillg" + - "khdang" + - "adityasa" + - "sboshra" + - "FabianMeiswinkel" + - "leminh98" + - "neildsh" + + step_4_monitor_ci: + description: "Monitor the azure-pipelines-msdata-direct.yml pipeline" + commands: + - "gh pr checks " + action: "Wait for CI to complete, investigate failures if any" + pipeline: "azure-pipelines-msdata-direct.yml" + + step_5_mark_ready: + condition: "All CI checks pass" + command: "gh pr ready " + action: "Convert from draft to ready for review" +``` + +--- + +## 3. Helper Script + +A companion PowerShell script is available at `tools/msdata-direct-sync-helper.ps1` to automate the mechanical parts of this workflow. + +```yaml +helper_script: + path: "tools/msdata-direct-sync-helper.ps1" + + usage: | + # Full automated workflow + .\tools\msdata-direct-sync-helper.ps1 -MsdataRepoPath "Q:\CosmosDB" + + # Individual phases + .\tools\msdata-direct-sync-helper.ps1 -MsdataRepoPath "Q:\CosmosDB" -Phase Setup + .\tools\msdata-direct-sync-helper.ps1 -MsdataRepoPath "Q:\CosmosDB" -Phase Branch + .\tools\msdata-direct-sync-helper.ps1 -MsdataRepoPath "Q:\CosmosDB" -Phase Sync + .\tools\msdata-direct-sync-helper.ps1 -MsdataRepoPath "Q:\CosmosDB" -Phase Build + .\tools\msdata-direct-sync-helper.ps1 -MsdataRepoPath "Q:\CosmosDB" -Phase PR + + parameters: + MsdataRepoPath: "Path to local msdata CosmosDB repo clone (required)" + Phase: "Run a specific phase only (optional; default runs all)" + GitHubUsername: "GitHub username for branch naming (optional; auto-detected)" + SkipBuild: "Skip build validation phase (optional; not recommended)" +``` + +--- + +## 4. Troubleshooting + +### Common Merge Conflicts + +```yaml +merge_conflicts: + directory_build_props: + description: "Version number conflicts between master and msdata/direct" + resolution: "Accept master version numbers, they are the source of truth" + + csproj_files: + description: "Project file conflicts from both sides adding references" + resolution: "Manually merge to include all references from both sides" + + direct_files: + description: "Files in src/direct/ modified on both branches" + resolution: "Accept msdata/direct version — these will be overwritten by msdata_sync.ps1" +``` + +### Build Failures After Sync + +```yaml +build_failures: + missing_type_or_namespace: + symptom: "error CS0246: The type or namespace name 'X' could not be found" + causes: + - "File not copied by msdata_sync.ps1" + - "New file added in msdata but not in sync script" + fix: "Locate file in msdata repo, copy manually to src/direct/" + + duplicate_definitions: + symptom: "error CS0111: Type already defines a member" + causes: + - "Same file exists in both master and direct" + fix: "Remove the duplicate, keep the direct version" + + api_incompatibility: + symptom: "error CS1501: No overload for method" + causes: + - "Direct package API changed" + fix: "Update calling code to match new API signature" +``` + +### msdata_sync.ps1 Errors + +```yaml +sync_script_errors: + file_not_found: + symptom: "Write-Error: False" + cause: "File exists in sync script list but not in msdata repo" + fix: "File may have been renamed or removed — check msdata repo history" + + permission_denied: + symptom: "Access to the path is denied" + cause: "File is read-only or locked" + fix: "Close any editors, remove read-only flag: attrib -r " + + path_not_found: + symptom: "Cannot find path 'Q:\\CosmosDB\\...'" + cause: "$baseDir not set correctly" + fix: "Verify msdata repo path and update $baseDir in script" +``` + +--- + +## 5. Sample Pull Requests + +Reference these PRs for expected format and scope: + +| PR | Title | Date | Scope | +|----|-------|------|-------| +| [#5612](https://github.com/Azure/azure-cosmos-dotnet-v3/pull/5612) | [Internal] Direct package: Adds msdata/direct update from master | Feb 2026 | 545 files, 76K additions | +| [#3776](https://github.com/Azure/azure-cosmos-dotnet-v3/pull/3776) | [Internal] Msdata/Direct: Refactors msdata/direct with v3 master and Direct v3.30.4 | Mar 2023 | 155 files, 13K additions | +| [#3726](https://github.com/Azure/azure-cosmos-dotnet-v3/pull/3726) | [Internal] Msdata/Direct: Refactors msdata branch with latest v3 and direct release | Feb 2023 | 361 files, 27K additions | + +--- + +## 6. CI Pipeline + +The `azure-pipelines-msdata-direct.yml` pipeline runs automatically on PRs targeting `msdata/direct*` branches. It includes: + +```yaml +ci_pipeline: + trigger: "PR to msdata/direct*" + jobs: + - "Static analysis tools" + - "CTL build" + - "Samples build" + - "msdata test suite (Release)" + - "Internal build" + - "Preview msdata build" + - "Thin client build" + variables: + test_filter: '--filter "TestCategory!=Flaky & TestCategory!=Quarantine & TestCategory!=Functional"' + vm_image: "windows-latest" +``` diff --git a/Microsoft.Azure.Cosmos/src/GatewayStoreModel.cs b/Microsoft.Azure.Cosmos/src/GatewayStoreModel.cs index 3123eed0ba..db8a549387 100644 --- a/Microsoft.Azure.Cosmos/src/GatewayStoreModel.cs +++ b/Microsoft.Azure.Cosmos/src/GatewayStoreModel.cs @@ -101,10 +101,10 @@ await GatewayStoreModel.ApplySessionTokenAsync( request.RequestContext.RegionName = regionName; } - bool isPPAFEnabled = this.IsPartitionLevelFailoverEnabled(); - // This is applicable for both per partition automatic failover and per partition circuit breaker. - if ((isPPAFEnabled || this.isThinClientEnabled) - && !ReplicatedResourceClient.IsMasterResource(request.ResourceType) + bool isPPAFEnabled = this.IsPartitionLevelFailoverEnabled(); + // This is applicable for both per partition automatic failover and per partition circuit breaker. + if ((isPPAFEnabled || this.isThinClientEnabled) + && !ReplicatedResourceClient.IsMasterResource(request.ResourceType) && (request.ResourceType.IsPartitioned() || request.ResourceType == ResourceType.StoredProcedure)) { (bool isSuccess, PartitionKeyRange partitionKeyRange) = await TryResolvePartitionKeyRangeAsync( @@ -563,7 +563,8 @@ internal static bool IsOperationSupportedByThinClient(DocumentServiceRequest req || request.OperationType == OperationType.Upsert || request.OperationType == OperationType.Replace || request.OperationType == OperationType.Delete - || request.OperationType == OperationType.Query)) + || request.OperationType == OperationType.Query + || request.OperationType == OperationType.QueryPlan)) { return true; } diff --git a/Microsoft.Azure.Cosmos/src/Query/v3Query/CosmosQueryClientCore.cs b/Microsoft.Azure.Cosmos/src/Query/v3Query/CosmosQueryClientCore.cs index 8f3c7ca366..d210dae18d 100644 --- a/Microsoft.Azure.Cosmos/src/Query/v3Query/CosmosQueryClientCore.cs +++ b/Microsoft.Azure.Cosmos/src/Query/v3Query/CosmosQueryClientCore.cs @@ -173,8 +173,8 @@ public override async Task> ExecuteItemQueryAsync( resourceType, message, trace); - } - + } + public override async Task ExecuteQueryPlanRequestAsync( string resourceUri, ResourceType resourceType, @@ -209,7 +209,20 @@ public override async Task ExecuteQueryPlanReques { // Syntax exception are argument exceptions and thrown to the user. message.EnsureSuccessStatusCode(); - partitionedQueryExecutionInfo = this.clientContext.SerializerCore.FromStream(message.Content); + + if (this.documentClient.isThinClientEnabled) + { + ContainerProperties containerProperties = await this.clientContext.GetCachedContainerPropertiesAsync( + resourceUri, trace, cancellationToken); + + partitionedQueryExecutionInfo = ThinClientQueryPlanHelper.DeserializeQueryPlanResponse( + message.Content, + containerProperties.PartitionKey); + } + else + { + partitionedQueryExecutionInfo = this.clientContext.SerializerCore.FromStream(message.Content); + } } return partitionedQueryExecutionInfo; diff --git a/Microsoft.Azure.Cosmos/src/SqlObjects/SqlFunctionCallScalarExpression.cs b/Microsoft.Azure.Cosmos/src/SqlObjects/SqlFunctionCallScalarExpression.cs index 01ef4e60e5..33bd903090 100644 --- a/Microsoft.Azure.Cosmos/src/SqlObjects/SqlFunctionCallScalarExpression.cs +++ b/Microsoft.Azure.Cosmos/src/SqlObjects/SqlFunctionCallScalarExpression.cs @@ -102,7 +102,9 @@ sealed class SqlFunctionCallScalarExpression : SqlScalarExpression { Names.IsNumber, Identifiers.IsNumber }, { Names.IsObject, Identifiers.IsObject }, { Names.IsPrimitive, Identifiers.IsPrimitive }, - { Names.IsString, Identifiers.IsString }, + { Names.IsString, Identifiers.IsString }, + { Names.LastSubstringAfter, Identifiers.LastSubstringAfter }, + { Names.LastSubstringBefore, Identifiers.LastSubstringBefore }, { Names.Left, Identifiers.Left }, { Names.Length, Identifiers.Length }, { Names.Like, Identifiers.Like }, @@ -143,7 +145,9 @@ sealed class SqlFunctionCallScalarExpression : SqlScalarExpression { Names.StringToNull, Identifiers.StringToNull }, { Names.StringToNumber, Identifiers.StringToNumber }, { Names.StringToObject, Identifiers.StringToObject }, - { Names.Substring, Identifiers.Substring }, + { Names.Substring, Identifiers.Substring }, + { Names.SubstringAfter, Identifiers.SubstringAfter }, + { Names.SubstringBefore, Identifiers.SubstringBefore }, { Names.Sum, Identifiers.Sum }, { Names.Tan, Identifiers.Tan }, { Names.TicksToDateTime, Identifiers.TicksToDateTime }, @@ -350,7 +354,9 @@ public static class Names public const string IsObject = "IS_OBJECT"; public const string IsPrimitive = "IS_PRIMITIVE"; public const string IsString = "IS_STRING"; - public const string LastIndexOf = "LastIndexOf"; + public const string LastIndexOf = "LastIndexOf"; + public const string LastSubstringAfter = "LastSubstringAfter"; + public const string LastSubstringBefore = "LastSubstringBefore"; public const string Left = "LEFT"; public const string Length = "LENGTH"; public const string Like = "LIKE"; @@ -398,7 +404,9 @@ public static class Names public const string StringToNull = "StringToNull"; public const string StringToNumber = "StringToNumber"; public const string StringToObject = "StringToObject"; - public const string Substring = "SUBSTRING"; + public const string Substring = "SUBSTRING"; + public const string SubstringAfter = "SubstringAfter"; + public const string SubstringBefore = "SubstringBefore"; public const string Sum = "SUM"; public const string Tan = "TAN"; public const string TicksToDateTime = "TicksToDateTime"; @@ -528,7 +536,9 @@ public static class Identifiers public static readonly SqlIdentifier IsObject = SqlIdentifier.Create(Names.IsObject); public static readonly SqlIdentifier IsPrimitive = SqlIdentifier.Create(Names.IsPrimitive); public static readonly SqlIdentifier IsString = SqlIdentifier.Create(Names.IsString); - public static readonly SqlIdentifier LastIndexOf = SqlIdentifier.Create(Names.LastIndexOf); + public static readonly SqlIdentifier LastIndexOf = SqlIdentifier.Create(Names.LastIndexOf); + public static readonly SqlIdentifier LastSubstringAfter = SqlIdentifier.Create(Names.LastSubstringAfter); + public static readonly SqlIdentifier LastSubstringBefore = SqlIdentifier.Create(Names.LastSubstringBefore); public static readonly SqlIdentifier Left = SqlIdentifier.Create(Names.Left); public static readonly SqlIdentifier Length = SqlIdentifier.Create(Names.Length); public static readonly SqlIdentifier Like = SqlIdentifier.Create(Names.Like); @@ -576,7 +586,9 @@ public static class Identifiers public static readonly SqlIdentifier StringToNull = SqlIdentifier.Create(Names.StringToNull); public static readonly SqlIdentifier StringToNumber = SqlIdentifier.Create(Names.StringToNumber); public static readonly SqlIdentifier StringToObject = SqlIdentifier.Create(Names.StringToObject); - public static readonly SqlIdentifier Substring = SqlIdentifier.Create(Names.Substring); + public static readonly SqlIdentifier Substring = SqlIdentifier.Create(Names.Substring); + public static readonly SqlIdentifier SubstringAfter = SqlIdentifier.Create(Names.SubstringAfter); + public static readonly SqlIdentifier SubstringBefore = SqlIdentifier.Create(Names.SubstringBefore); public static readonly SqlIdentifier Sum = SqlIdentifier.Create(Names.Sum); public static readonly SqlIdentifier Tan = SqlIdentifier.Create(Names.Tan); public static readonly SqlIdentifier TicksToDateTime = SqlIdentifier.Create(Names.TicksToDateTime); diff --git a/Microsoft.Azure.Cosmos/src/ThinClientQueryPlanHelper.cs b/Microsoft.Azure.Cosmos/src/ThinClientQueryPlanHelper.cs new file mode 100644 index 0000000000..6edb6c21a0 --- /dev/null +++ b/Microsoft.Azure.Cosmos/src/ThinClientQueryPlanHelper.cs @@ -0,0 +1,162 @@ +//------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +//------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Query.Core.QueryPlan +{ + using System; + using System.Collections.Generic; + using System.IO; + using System.Text.Json; + using Newtonsoft.Json; + using PartitionKeyDefinition = Documents.PartitionKeyDefinition; + using PartitionKeyInternal = Documents.Routing.PartitionKeyInternal; + + /// + /// Handles conversion of thin client query plan responses where query ranges + /// are returned in PartitionKeyInternal format instead of EPK hex strings. + /// Mirrors the conversion logic in . + /// + /// + /// Uses System.Text.Json for primary parsing and structural validation. + /// Newtonsoft.Json is used only for deserializing QueryInfo, HybridSearchQueryInfo, + /// and Range<PartitionKeyInternal> because these types and their deep type hierarchies + /// (including the external Direct package types) use Newtonsoft [JsonProperty] attributes + /// and [JsonObject(MemberSerialization.OptIn)] semantics that have no System.Text.Json equivalent. + /// + internal static class ThinClientQueryPlanHelper + { + private static readonly Newtonsoft.Json.JsonSerializerSettings NewtonsoftSettings = + new Newtonsoft.Json.JsonSerializerSettings + { + DateParseHandling = Newtonsoft.Json.DateParseHandling.None, + MaxDepth = 64, + }; + + /// + /// Deserializes a thin client query plan response stream into a + /// with EPK string ranges. + /// The response contains query ranges in PartitionKeyInternal format + /// which are converted to EPK hex strings and sorted. + /// + /// The response stream containing the raw query plan JSON. + /// The partition key definition for the container. + /// with sorted EPK string ranges. + /// Thrown when or is null. + /// Thrown when the response JSON is malformed or missing required properties. + public static PartitionedQueryExecutionInfo DeserializeQueryPlanResponse( + Stream stream, + PartitionKeyDefinition partitionKeyDefinition) + { + if (stream == null) + { + throw new ArgumentNullException(nameof(stream)); + } + + if (partitionKeyDefinition == null) + { + throw new ArgumentNullException(nameof(partitionKeyDefinition)); + } + + using JsonDocument doc = JsonDocument.Parse(stream); + JsonElement root = doc.RootElement; + + if (root.ValueKind != JsonValueKind.Object) + { + throw new FormatException( + $"Thin client query plan response must be a JSON object, but was {root.ValueKind}."); + } + + // Validate and extract queryRanges (required) + if (!root.TryGetProperty("queryRanges", out JsonElement queryRangesElement)) + { + throw new FormatException( + "Thin client query plan response is missing the required 'queryRanges' property."); + } + + if (queryRangesElement.ValueKind != JsonValueKind.Array) + { + throw new FormatException( + $"Expected 'queryRanges' to be a JSON array, but was {queryRangesElement.ValueKind}."); + } + + if (queryRangesElement.GetArrayLength() == 0) + { + throw new FormatException( + "Thin client query plan response 'queryRanges' array must not be empty."); + } + + // Deserialize QueryInfo using Newtonsoft because QueryInfo uses + // [JsonObject(MemberSerialization.OptIn)] and Newtonsoft-only [JsonProperty] attributes. + QueryInfo queryInfo = null; + if (root.TryGetProperty("queryInfo", out JsonElement queryInfoElement) + && queryInfoElement.ValueKind != JsonValueKind.Null) + { + queryInfo = Newtonsoft.Json.JsonConvert.DeserializeObject( + queryInfoElement.GetRawText(), + ThinClientQueryPlanHelper.NewtonsoftSettings); + } + + // Deserialize HybridSearchQueryInfo using Newtonsoft (same constraint as QueryInfo). + HybridSearchQueryInfo hybridSearchQueryInfo = null; + if (root.TryGetProperty("hybridSearchQueryInfo", out JsonElement hybridElement) + && hybridElement.ValueKind != JsonValueKind.Null) + { + hybridSearchQueryInfo = Newtonsoft.Json.JsonConvert.DeserializeObject( + hybridElement.GetRawText(), + ThinClientQueryPlanHelper.NewtonsoftSettings); + } + + // Parse and convert query ranges to EPK string ranges. + // Range requires Newtonsoft because PartitionKeyInternal + // is from the external Direct package with Newtonsoft-based serialization. + List> effectiveRanges = + new List>(queryRangesElement.GetArrayLength()); + + foreach (JsonElement rangeElement in queryRangesElement.EnumerateArray()) + { + if (rangeElement.ValueKind != JsonValueKind.Object) + { + throw new FormatException( + $"Each query range must be a JSON object, but was {rangeElement.ValueKind}."); + } + + if (!rangeElement.TryGetProperty("min", out _)) + { + throw new FormatException( + "Query range is missing the required 'min' property."); + } + + if (!rangeElement.TryGetProperty("max", out _)) + { + throw new FormatException( + "Query range is missing the required 'max' property."); + } + + Documents.Routing.Range internalRange = + Newtonsoft.Json.JsonConvert.DeserializeObject>( + rangeElement.GetRawText(), + ThinClientQueryPlanHelper.NewtonsoftSettings); + + if (internalRange == null) + { + throw new FormatException( + "Failed to deserialize query range from thin client response."); + } + + effectiveRanges.Add(PartitionKeyInternal.GetEffectivePartitionKeyRange( + partitionKeyDefinition, + internalRange)); + } + + effectiveRanges.Sort(Documents.Routing.Range.MinComparer.Instance); + + return new PartitionedQueryExecutionInfo() + { + QueryInfo = queryInfo, + QueryRanges = effectiveRanges, + HybridSearchQueryInfo = hybridSearchQueryInfo, + }; + } + } +} \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosItemThinClientTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosItemThinClientTests.cs index 7cbd053101..d34866a6ca 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosItemThinClientTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosItemThinClientTests.cs @@ -31,12 +31,12 @@ public class CosmosItemThinClientTests private MultiRegionSetupHelpers.CosmosSystemTextJsonSerializer cosmosSystemTextJsonSerializer; private const int ItemCount = 100; - [TestInitialize] - public async Task TestInitAsync() - { - Environment.SetEnvironmentVariable(ConfigurationManager.ThinClientModeEnabled, "True"); + [TestInitialize] + public async Task TestInitAsync() + { + Environment.SetEnvironmentVariable(ConfigurationManager.ThinClientModeEnabled, "True"); this.connectionString = Environment.GetEnvironmentVariable("COSMOSDB_THINCLIENT"); - + if (string.IsNullOrEmpty(this.connectionString)) { Assert.Fail("Set environment variable COSMOSDB_THINCLIENT to run the tests"); @@ -188,183 +188,227 @@ public async Task RegionalDatabaseAccountNameIsEmptyInPayload() [TestCategory("ThinClient")] public async Task TestThinClientWithExecuteStoredProcedureAsync() { - Environment.SetEnvironmentVariable(ConfigurationManager.ThinClientModeEnabled, "true"); + CosmosClient localClient = null; + Database localDatabase = null; - JsonSerializerOptions jsonSerializerOptions = new JsonSerializerOptions + try { - PropertyNamingPolicy = null, - PropertyNameCaseInsensitive = true, - DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull - }; - this.cosmosSystemTextJsonSerializer = new MultiRegionSetupHelpers.CosmosSystemTextJsonSerializer(jsonSerializerOptions); + Environment.SetEnvironmentVariable(ConfigurationManager.ThinClientModeEnabled, "true"); - this.client = new CosmosClient( - this.connectionString, - new CosmosClientOptions() - { - ConnectionMode = ConnectionMode.Gateway, - Serializer = this.cosmosSystemTextJsonSerializer, - }); + JsonSerializerOptions jsonSerializerOptions = new JsonSerializerOptions + { + PropertyNamingPolicy = null, + PropertyNameCaseInsensitive = true, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull + }; + CosmosSystemTextJsonSerializer localSerializer = new MultiRegionSetupHelpers.CosmosSystemTextJsonSerializer(jsonSerializerOptions); - string uniqueDbName = "TestDbStoreProc_" + Guid.NewGuid().ToString(); - this.database = await this.client.CreateDatabaseIfNotExistsAsync(uniqueDbName); - string uniqueContainerName = "TestDbStoreProcContainer_" + Guid.NewGuid().ToString(); - this.container = await this.database.CreateContainerIfNotExistsAsync(uniqueContainerName, "/pk"); + localClient = new CosmosClient( + this.connectionString, + new CosmosClientOptions() + { + ConnectionMode = ConnectionMode.Gateway, + Serializer = localSerializer, + }); + string uniqueDbName = "TestDbStoreProc_" + Guid.NewGuid().ToString(); + localDatabase = await localClient.CreateDatabaseIfNotExistsAsync(uniqueDbName); + string uniqueContainerName = "TestDbStoreProcContainer_" + Guid.NewGuid().ToString(); + Container localContainer = await localDatabase.CreateContainerIfNotExistsAsync(uniqueContainerName, "/pk"); - string sprocId = "testSproc_" + Guid.NewGuid().ToString(); - string sprocBody = @"function(itemToCreate) { - var context = getContext(); - var collection = context.getCollection(); - var response = context.getResponse(); + + string sprocId = "testSproc_" + Guid.NewGuid().ToString(); + string sprocBody = @"function(itemToCreate) { + var context = getContext(); + var collection = context.getCollection(); + var response = context.getResponse(); - if (!itemToCreate) throw new Error('Item is undefined or null.'); + if (!itemToCreate) throw new Error('Item is undefined or null.'); - // Create a document - var accepted = collection.createDocument( - collection.getSelfLink(), - itemToCreate, - function(err, newItem) { - if (err) throw err; - - // Query the created document - var query = 'SELECT * FROM c WHERE c.id = ""' + newItem.id + '""'; - var isAccepted = collection.queryDocuments( + // Create a document + var accepted = collection.createDocument( collection.getSelfLink(), - query, - function(queryErr, documents) { - if (queryErr) throw queryErr; - response.setBody({ - created: newItem, - queried: documents[0] - }); - } - ); - if (!isAccepted) throw 'Query not accepted'; - }); + itemToCreate, + function(err, newItem) { + if (err) throw err; + + // Query the created document + var query = 'SELECT * FROM c WHERE c.id = ""' + newItem.id + '""'; + var isAccepted = collection.queryDocuments( + collection.getSelfLink(), + query, + function(queryErr, documents) { + if (queryErr) throw queryErr; + response.setBody({ + created: newItem, + queried: documents[0] + }); + } + ); + if (!isAccepted) throw 'Query not accepted'; + }); - if (!accepted) throw new Error('Create was not accepted.'); - }"; + if (!accepted) throw new Error('Create was not accepted.'); + }"; - // Create stored procedure - Scripts.StoredProcedureResponse createResponse = await this.container.Scripts.CreateStoredProcedureAsync( - new Scripts.StoredProcedureProperties(sprocId, sprocBody)); - Assert.AreEqual(HttpStatusCode.Created, createResponse.StatusCode); + // Create stored procedure + Scripts.StoredProcedureResponse createResponse = await localContainer.Scripts.CreateStoredProcedureAsync( + new Scripts.StoredProcedureProperties(sprocId, sprocBody)); + Assert.AreEqual(HttpStatusCode.Created, createResponse.StatusCode); - // Execute stored procedure - string testPartitionId = Guid.NewGuid().ToString(); - TestObject testItem = new TestObject - { - Id = Guid.NewGuid().ToString(), - Pk = testPartitionId, - Other = "Created by Stored Procedure" - }; + // Execute stored procedure + string testPartitionId = Guid.NewGuid().ToString(); + TestObject testItem = new TestObject + { + Id = Guid.NewGuid().ToString(), + Pk = testPartitionId, + Other = "Created by Stored Procedure" + }; - Scripts.StoredProcedureExecuteResponse executeResponse = - await this.container.Scripts.ExecuteStoredProcedureAsync( - sprocId, - new PartitionKey(testPartitionId), - new dynamic[] { testItem }); + Scripts.StoredProcedureExecuteResponse executeResponse = + await localContainer.Scripts.ExecuteStoredProcedureAsync( + sprocId, + new PartitionKey(testPartitionId), + new dynamic[] { testItem }); - Assert.AreEqual(HttpStatusCode.OK, executeResponse.StatusCode); - Assert.IsNotNull(executeResponse.Resource); - string diagnostics = executeResponse.Diagnostics.ToString(); - Assert.IsTrue(diagnostics.Contains("|F4"), "Diagnostics User Agent should contain '|F4' for ThinClient"); + Assert.AreEqual(HttpStatusCode.OK, executeResponse.StatusCode); + Assert.IsNotNull(executeResponse.Resource); + string diagnostics = executeResponse.Diagnostics.ToString(); + Assert.IsTrue(diagnostics.Contains("|F4"), "Diagnostics User Agent should contain '|F4' for ThinClient"); - // Delete stored procedure - await this.container.Scripts.DeleteStoredProcedureAsync(sprocId); + // Delete stored procedure + await localContainer.Scripts.DeleteStoredProcedureAsync(sprocId); + } + finally + { + if (localDatabase != null) + { + try + { + await localDatabase.DeleteAsync(); + } + catch { } + } + + if (localClient != null) + { + localClient.Dispose(); + } + } } - [TestMethod] - [TestCategory("ThinClient")] - public async Task TestThinClientWithExecuteStoredProcedureStreamAsync() + [TestMethod] + [TestCategory("ThinClient")] + public async Task TestThinClientWithExecuteStoredProcedureStreamAsync() { - Environment.SetEnvironmentVariable(ConfigurationManager.ThinClientModeEnabled, "true"); - - JsonSerializerOptions jsonSerializerOptions = new JsonSerializerOptions - { - PropertyNamingPolicy = null, - PropertyNameCaseInsensitive = true, - DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull - }; - this.cosmosSystemTextJsonSerializer = new MultiRegionSetupHelpers.CosmosSystemTextJsonSerializer(jsonSerializerOptions); - - this.client = new CosmosClient( - this.connectionString, - new CosmosClientOptions() - { - ConnectionMode = ConnectionMode.Gateway, - Serializer = this.cosmosSystemTextJsonSerializer, - }); - - string uniqueDbName = "TestDbStoreProc_" + Guid.NewGuid().ToString(); - this.database = await this.client.CreateDatabaseIfNotExistsAsync(uniqueDbName); - string uniqueContainerName = "TestDbStoreProcContainer_" + Guid.NewGuid().ToString(); - this.container = await this.database.CreateContainerIfNotExistsAsync(uniqueContainerName, "/pk"); - - - string sprocId = "testSproc_" + Guid.NewGuid().ToString(); - string sprocBody = @"function(itemToCreate) { - var context = getContext(); - var collection = context.getCollection(); - var response = context.getResponse(); - - if (!itemToCreate) throw new Error('Item is undefined or null.'); - - // Create a document - var accepted = collection.createDocument( - collection.getSelfLink(), - itemToCreate, - function(err, newItem) { - if (err) throw err; - - // Query the created document - var query = 'SELECT * FROM c WHERE c.id = ""' + newItem.id + '""'; - var isAccepted = collection.queryDocuments( - collection.getSelfLink(), - query, - function(queryErr, documents) { - if (queryErr) throw queryErr; - response.setBody({ - created: newItem, - queried: documents[0] - }); - } - ); - if (!isAccepted) throw 'Query not accepted'; - }); - - if (!accepted) throw new Error('Create was not accepted.'); - }"; - - // Create stored procedure - Scripts.StoredProcedureResponse createResponse = await this.container.Scripts.CreateStoredProcedureAsync( - new Scripts.StoredProcedureProperties(sprocId, sprocBody)); - Assert.AreEqual(HttpStatusCode.Created, createResponse.StatusCode); - - // Execute stored procedure - string testPartitionId = Guid.NewGuid().ToString(); - TestObject testItem = new TestObject - { - Id = Guid.NewGuid().ToString(), - Pk = testPartitionId, - Other = "Created by Stored Procedure" - }; - - using (ResponseMessage executeResponse = - await this.container.Scripts.ExecuteStoredProcedureStreamAsync( - sprocId, - new PartitionKey(testPartitionId), - new dynamic[] { testItem })) - { - Assert.AreEqual(HttpStatusCode.OK, executeResponse.StatusCode); - Assert.IsNotNull(executeResponse.Content); - string diagnostics = executeResponse.Diagnostics.ToString(); - Assert.IsTrue(diagnostics.Contains("|F4"), "Diagnostics User Agent should contain '|F4' for ThinClient"); - } - - // Delete stored procedure - await this.container.Scripts.DeleteStoredProcedureAsync(sprocId); + CosmosClient localClient = null; + Database localDatabase = null; + + try + { + Environment.SetEnvironmentVariable(ConfigurationManager.ThinClientModeEnabled, "true"); + + JsonSerializerOptions jsonSerializerOptions = new JsonSerializerOptions + { + PropertyNamingPolicy = null, + PropertyNameCaseInsensitive = true, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull + }; + CosmosSystemTextJsonSerializer localSerializer = new MultiRegionSetupHelpers.CosmosSystemTextJsonSerializer(jsonSerializerOptions); + + localClient = new CosmosClient( + this.connectionString, + new CosmosClientOptions() + { + ConnectionMode = ConnectionMode.Gateway, + Serializer = localSerializer, + }); + + string uniqueDbName = "TestDbStoreProc_" + Guid.NewGuid().ToString(); + localDatabase = await localClient.CreateDatabaseIfNotExistsAsync(uniqueDbName); + string uniqueContainerName = "TestDbStoreProcContainer_" + Guid.NewGuid().ToString(); + Container localContainer = await localDatabase.CreateContainerIfNotExistsAsync(uniqueContainerName, "/pk"); + + + string sprocId = "testSproc_" + Guid.NewGuid().ToString(); + string sprocBody = @"function(itemToCreate) { + var context = getContext(); + var collection = context.getCollection(); + var response = context.getResponse(); + + if (!itemToCreate) throw new Error('Item is undefined or null.'); + + // Create a document + var accepted = collection.createDocument( + collection.getSelfLink(), + itemToCreate, + function(err, newItem) { + if (err) throw err; + + // Query the created document + var query = 'SELECT * FROM c WHERE c.id = ""' + newItem.id + '""'; + var isAccepted = collection.queryDocuments( + collection.getSelfLink(), + query, + function(queryErr, documents) { + if (queryErr) throw queryErr; + response.setBody({ + created: newItem, + queried: documents[0] + }); + } + ); + if (!isAccepted) throw 'Query not accepted'; + }); + + if (!accepted) throw new Error('Create was not accepted.'); + }"; + + // Create stored procedure + Scripts.StoredProcedureResponse createResponse = await localContainer.Scripts.CreateStoredProcedureAsync( + new Scripts.StoredProcedureProperties(sprocId, sprocBody)); + Assert.AreEqual(HttpStatusCode.Created, createResponse.StatusCode); + + // Execute stored procedure + string testPartitionId = Guid.NewGuid().ToString(); + TestObject testItem = new TestObject + { + Id = Guid.NewGuid().ToString(), + Pk = testPartitionId, + Other = "Created by Stored Procedure" + }; + + using (ResponseMessage executeResponse = + await localContainer.Scripts.ExecuteStoredProcedureStreamAsync( + sprocId, + new PartitionKey(testPartitionId), + new dynamic[] { testItem })) + { + Assert.AreEqual(HttpStatusCode.OK, executeResponse.StatusCode); + Assert.IsNotNull(executeResponse.Content); + string diagnostics = executeResponse.Diagnostics.ToString(); + Assert.IsTrue(diagnostics.Contains("|F4"), "Diagnostics User Agent should contain '|F4' for ThinClient"); + } + + // Delete stored procedure + await localContainer.Scripts.DeleteStoredProcedureAsync(sprocId); + } + finally + { + if (localDatabase != null) + { + try + { + await localDatabase.DeleteAsync(); + } + catch { } + } + + if (localClient != null) + { + localClient.Dispose(); + } + } } [TestMethod] @@ -426,42 +470,65 @@ public async Task CreateItemsTest() [TestCategory("ThinClient")] public async Task CreateItemsTestWithThinClientFlagEnabledAndAccountDisabled() { - Environment.SetEnvironmentVariable(ConfigurationManager.ThinClientModeEnabled, "True"); - string authKey = Utils.ConfigurationManager.AppSettings["MasterKey"]; - string endpoint = Utils.ConfigurationManager.AppSettings["GatewayEndpoint"]; - AzureKeyCredential masterKeyCredential = new AzureKeyCredential(authKey); + CosmosClient localClient = null; + Database localDatabase = null; + Container localContainer = null; - JsonSerializerOptions jsonSerializerOptions = new JsonSerializerOptions + try { - PropertyNamingPolicy = null, - PropertyNameCaseInsensitive = true, - DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull - }; - this.cosmosSystemTextJsonSerializer = new MultiRegionSetupHelpers.CosmosSystemTextJsonSerializer(jsonSerializerOptions); - - this.client = new CosmosClient( - endpoint, - masterKeyCredential, - new CosmosClientOptions() - { - ConnectionMode = ConnectionMode.Gateway, - Serializer = this.cosmosSystemTextJsonSerializer, - }); + Environment.SetEnvironmentVariable(ConfigurationManager.ThinClientModeEnabled, "True"); + string authKey = Utils.ConfigurationManager.AppSettings["MasterKey"]; + string endpoint = Utils.ConfigurationManager.AppSettings["GatewayEndpoint"]; + AzureKeyCredential masterKeyCredential = new AzureKeyCredential(authKey); - string uniqueDbName = "TestDb2_" + Guid.NewGuid().ToString(); - this.database = await this.client.CreateDatabaseIfNotExistsAsync(uniqueDbName); - string uniqueContainerName = "TestContainer2_" + Guid.NewGuid().ToString(); - this.container = await this.database.CreateContainerIfNotExistsAsync(uniqueContainerName, "/pk"); - - string pk = "pk_create"; - IEnumerable items = this.GenerateItems(pk); - - foreach (TestObject item in items) + JsonSerializerOptions jsonSerializerOptions = new JsonSerializerOptions + { + PropertyNamingPolicy = null, + PropertyNameCaseInsensitive = true, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull + }; + CosmosSystemTextJsonSerializer localSerializer = new MultiRegionSetupHelpers.CosmosSystemTextJsonSerializer(jsonSerializerOptions); + + localClient = new CosmosClient( + endpoint, + masterKeyCredential, + new CosmosClientOptions() + { + ConnectionMode = ConnectionMode.Gateway, + Serializer = localSerializer, + }); + + string uniqueDbName = "TestDb2_" + Guid.NewGuid().ToString(); + localDatabase = await localClient.CreateDatabaseIfNotExistsAsync(uniqueDbName); + string uniqueContainerName = "TestContainer2_" + Guid.NewGuid().ToString(); + localContainer = await localDatabase.CreateContainerIfNotExistsAsync(uniqueContainerName, "/pk"); + + string pk = "pk_create"; + IEnumerable items = this.GenerateItems(pk); + + foreach (TestObject item in items) + { + ItemResponse response = await localContainer.CreateItemAsync(item, new PartitionKey(item.Pk)); + Assert.AreEqual(HttpStatusCode.Created, response.StatusCode); + string diagnostics = response.Diagnostics.ToString(); + Assert.IsFalse(diagnostics.Contains("|F4"), "Diagnostics User Agent should NOT contain '|F4' for Gateway"); + } + } + finally { - ItemResponse response = await this.container.CreateItemAsync(item, new PartitionKey(item.Pk)); - Assert.AreEqual(HttpStatusCode.Created, response.StatusCode); - string diagnostics = response.Diagnostics.ToString(); - Assert.IsFalse(diagnostics.Contains("|F4"), "Diagnostics User Agent should NOT contain '|F4' for Gateway"); + if (localDatabase != null) + { + try + { + await localDatabase.DeleteAsync(); + } + catch { } + } + + if (localClient != null) + { + localClient.Dispose(); + } } } @@ -469,42 +536,65 @@ public async Task CreateItemsTestWithThinClientFlagEnabledAndAccountDisabled() [TestCategory("ThinClient")] public async Task CreateItemsTestWithDirectMode_ThinClientFlagEnabledAndAccountEnabled() { - JsonSerializerOptions jsonSerializerOptions = new JsonSerializerOptions + CosmosClient localClient = null; + Database localDatabase = null; + Container localContainer = null; + + try { - PropertyNamingPolicy = null, - PropertyNameCaseInsensitive = true, - DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull - }; - this.cosmosSystemTextJsonSerializer = new MultiRegionSetupHelpers.CosmosSystemTextJsonSerializer(jsonSerializerOptions); + JsonSerializerOptions jsonSerializerOptions = new JsonSerializerOptions + { + PropertyNamingPolicy = null, + PropertyNameCaseInsensitive = true, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull + }; + CosmosSystemTextJsonSerializer localSerializer = new MultiRegionSetupHelpers.CosmosSystemTextJsonSerializer(jsonSerializerOptions); - this.client = new CosmosClient( - this.connectionString, - new CosmosClientOptions() - { - ConnectionMode = ConnectionMode.Direct, - Serializer = this.cosmosSystemTextJsonSerializer, - }); + localClient = new CosmosClient( + this.connectionString, + new CosmosClientOptions() + { + ConnectionMode = ConnectionMode.Direct, + Serializer = localSerializer, + }); - string uniqueDbName = "TestDb2_" + Guid.NewGuid().ToString(); - this.database = await this.client.CreateDatabaseIfNotExistsAsync(uniqueDbName); - string uniqueContainerName = "TestContainer2_" + Guid.NewGuid().ToString(); - this.container = await this.database.CreateContainerIfNotExistsAsync(uniqueContainerName, "/pk"); + string uniqueDbName = "TestDb2_" + Guid.NewGuid().ToString(); + localDatabase = await localClient.CreateDatabaseIfNotExistsAsync(uniqueDbName); + string uniqueContainerName = "TestContainer2_" + Guid.NewGuid().ToString(); + localContainer = await localDatabase.CreateContainerIfNotExistsAsync(uniqueContainerName, "/pk"); - string pk = "pk_create"; - IEnumerable items = this.GenerateItems(pk); + string pk = "pk_create"; + IEnumerable items = this.GenerateItems(pk); - foreach (TestObject item in items) + foreach (TestObject item in items) + { + ItemResponse response = await localContainer.CreateItemAsync(item, new PartitionKey(item.Pk)); + Assert.AreEqual(HttpStatusCode.Created, response.StatusCode); + JsonDocument doc = JsonDocument.Parse(response.Diagnostics.ToString()); + string connectionMode = doc.RootElement + .GetProperty("data") + .GetProperty("Client Configuration") + .GetProperty("ConnectionMode") + .GetString(); + + Assert.AreEqual("Direct", connectionMode, "Diagnostics should have ConnectionMode set to 'Direct'"); + } + } + finally { - ItemResponse response = await this.container.CreateItemAsync(item, new PartitionKey(item.Pk)); - Assert.AreEqual(HttpStatusCode.Created, response.StatusCode); - JsonDocument doc = JsonDocument.Parse(response.Diagnostics.ToString()); - string connectionMode = doc.RootElement - .GetProperty("data") - .GetProperty("Client Configuration") - .GetProperty("ConnectionMode") - .GetString(); - - Assert.AreEqual("Direct", connectionMode, "Diagnostics should have ConnectionMode set to 'Direct'"); + if (localDatabase != null) + { + try + { + await localDatabase.DeleteAsync(); + } + catch { } + } + + if (localClient != null) + { + localClient.Dispose(); + } } } @@ -512,38 +602,61 @@ public async Task CreateItemsTestWithDirectMode_ThinClientFlagEnabledAndAccountE [TestCategory("ThinClient")] public async Task CreateItemsTestWithThinClientFlagDisabledAccountEnabled() { - Environment.SetEnvironmentVariable(ConfigurationManager.ThinClientModeEnabled, "False"); + CosmosClient localClient = null; + Database localDatabase = null; + Container localContainer = null; - JsonSerializerOptions jsonSerializerOptions = new JsonSerializerOptions + try { - PropertyNamingPolicy = null, - PropertyNameCaseInsensitive = true, - DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull - }; - this.cosmosSystemTextJsonSerializer = new MultiRegionSetupHelpers.CosmosSystemTextJsonSerializer(jsonSerializerOptions); + Environment.SetEnvironmentVariable(ConfigurationManager.ThinClientModeEnabled, "False"); - this.client = new CosmosClient( - this.connectionString, - new CosmosClientOptions() - { - ConnectionMode = ConnectionMode.Gateway, - Serializer = this.cosmosSystemTextJsonSerializer, - }); + JsonSerializerOptions jsonSerializerOptions = new JsonSerializerOptions + { + PropertyNamingPolicy = null, + PropertyNameCaseInsensitive = true, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull + }; + CosmosSystemTextJsonSerializer localSerializer = new MultiRegionSetupHelpers.CosmosSystemTextJsonSerializer(jsonSerializerOptions); - string uniqueDbName = "TestDbTCDisabled_" + Guid.NewGuid().ToString(); - this.database = await this.client.CreateDatabaseIfNotExistsAsync(uniqueDbName); - string uniqueContainerName = "TestContainerTCDisabled_" + Guid.NewGuid().ToString(); - this.container = await this.database.CreateContainerIfNotExistsAsync(uniqueContainerName, "/pk"); + localClient = new CosmosClient( + this.connectionString, + new CosmosClientOptions() + { + ConnectionMode = ConnectionMode.Gateway, + Serializer = localSerializer, + }); - string pk = "pk_create"; - IEnumerable items = this.GenerateItems(pk); + string uniqueDbName = "TestDbTCDisabled_" + Guid.NewGuid().ToString(); + localDatabase = await localClient.CreateDatabaseIfNotExistsAsync(uniqueDbName); + string uniqueContainerName = "TestContainerTCDisabled_" + Guid.NewGuid().ToString(); + localContainer = await localDatabase.CreateContainerIfNotExistsAsync(uniqueContainerName, "/pk"); - foreach (TestObject item in items) + string pk = "pk_create"; + IEnumerable items = this.GenerateItems(pk); + + foreach (TestObject item in items) + { + ItemResponse response = await localContainer.CreateItemAsync(item, new PartitionKey(item.Pk)); + Assert.AreEqual(HttpStatusCode.Created, response.StatusCode); + string diagnostics = response.Diagnostics.ToString(); + Assert.IsFalse(diagnostics.Contains("|F4"), "Diagnostics User Agent should NOT contain '|F4' for Gateway"); + } + } + finally { - ItemResponse response = await this.container.CreateItemAsync(item, new PartitionKey(item.Pk)); - Assert.AreEqual(HttpStatusCode.Created, response.StatusCode); - string diagnostics = response.Diagnostics.ToString(); - Assert.IsFalse(diagnostics.Contains("|F4"), "Diagnostics User Agent should NOT contain '|F4' for Gateway"); + if (localDatabase != null) + { + try + { + await localDatabase.DeleteAsync(); + } + catch { } + } + + if (localClient != null) + { + localClient.Dispose(); + } } } @@ -764,77 +877,150 @@ public async Task QueryItemsTest() [TestCategory("ThinClient")] public async Task QueryItemsTestWithStrongConsistency() { - string connectionString = ConfigurationManager.GetEnvironmentVariable("COSMOSDB_THINCLIENTSTRONG", string.Empty); - if (string.IsNullOrEmpty(connectionString)) + CosmosClient localClient = null; + Database localDatabase = null; + + try { - Assert.Fail("Set environment variable COSMOSDB_THINCLIENTSTRONG to run the tests"); - } - this.client = new CosmosClient( - connectionString, - new CosmosClientOptions() - { - ConnectionMode = ConnectionMode.Gateway, - RequestTimeout = TimeSpan.FromSeconds(60), - ConsistencyLevel = Microsoft.Azure.Cosmos.ConsistencyLevel.Strong - }); - - string uniqueDbName = "TestDbTC_" + Guid.NewGuid().ToString(); - this.database = await this.client.CreateDatabaseIfNotExistsAsync(uniqueDbName); - string uniqueContainerName = "TestContainerTC_" + Guid.NewGuid().ToString(); - this.container = await this.database.CreateContainerIfNotExistsAsync(uniqueContainerName, "/pk"); + string connectionString = ConfigurationManager.GetEnvironmentVariable("COSMOSDB_THINCLIENTSTRONG", string.Empty); + if (string.IsNullOrEmpty(connectionString)) + { + Assert.Fail("Set environment variable COSMOSDB_THINCLIENTSTRONG to run the tests"); + } - string pk = "pk_query"; - List items = this.GenerateItems(pk).ToList(); + localClient = new CosmosClient( + connectionString, + new CosmosClientOptions() + { + ConnectionMode = ConnectionMode.Gateway, + RequestTimeout = TimeSpan.FromSeconds(60), + ConsistencyLevel = Microsoft.Azure.Cosmos.ConsistencyLevel.Strong + }); + + string uniqueDbName = "TestDbTC_" + Guid.NewGuid().ToString(); + localDatabase = await localClient.CreateDatabaseIfNotExistsAsync(uniqueDbName); + string uniqueContainerName = "TestContainerTC_" + Guid.NewGuid().ToString(); + Container localContainer = await localDatabase.CreateContainerIfNotExistsAsync(uniqueContainerName, "/pk"); + + string pk = "pk_query"; + List items = this.GenerateItems(pk).ToList(); + + List itemsCreated = new List(); + foreach (TestObject item in items) + { + try + { + ItemResponse response = await localContainer.CreateItemAsync(item, new PartitionKey(item.Pk)); + if (response.StatusCode == HttpStatusCode.Created) + { + itemsCreated.Add(item); + } + } + catch (CosmosException) + { + } + } - List createdItems = await this.CreateItemsSafeAsync(items); + string query = $"SELECT * FROM c WHERE c.pk = '{pk}'"; + FeedIterator iterator = localContainer.GetItemQueryIterator(query); - string query = $"SELECT * FROM c WHERE c.pk = '{pk}'"; - FeedIterator iterator = this.container.GetItemQueryIterator(query); + int count = 0; + while (iterator.HasMoreResults) + { + FeedResponse response = await iterator.ReadNextAsync(); + count += response.Count; + } - int count = 0; - while (iterator.HasMoreResults) - { - FeedResponse response = await iterator.ReadNextAsync(); - count += response.Count; + Assert.AreEqual(itemsCreated.Count, count); } + finally + { + if (localDatabase != null) + { + try + { + await localDatabase.DeleteAsync(); + } + catch { } + } - Assert.AreEqual(createdItems.Count, count); + if (localClient != null) + { + localClient.Dispose(); + } + } } [TestMethod] [TestCategory("ThinClient")] public async Task QueryItemsTestWithSessionConsistency() { - this.client = new CosmosClient( - this.connectionString, - new CosmosClientOptions() - { - ConnectionMode = ConnectionMode.Gateway, - RequestTimeout = TimeSpan.FromSeconds(60), - ConsistencyLevel = Microsoft.Azure.Cosmos.ConsistencyLevel.Session - }); - - string uniqueDbName = "TestDbTC_" + Guid.NewGuid().ToString(); - this.database = await this.client.CreateDatabaseIfNotExistsAsync(uniqueDbName); - string uniqueContainerName = "TestContainerTC_" + Guid.NewGuid().ToString(); - this.container = await this.database.CreateContainerIfNotExistsAsync(uniqueContainerName, "/pk"); + CosmosClient localClient = null; + Database localDatabase = null; - string pk = "pk_query"; - List items = this.GenerateItems(pk).ToList(); + try + { + localClient = new CosmosClient( + this.connectionString, + new CosmosClientOptions() + { + ConnectionMode = ConnectionMode.Gateway, + RequestTimeout = TimeSpan.FromSeconds(60), + ConsistencyLevel = Microsoft.Azure.Cosmos.ConsistencyLevel.Session + }); + + string uniqueDbName = "TestDbTC_" + Guid.NewGuid().ToString(); + localDatabase = await localClient.CreateDatabaseIfNotExistsAsync(uniqueDbName); + string uniqueContainerName = "TestContainerTC_" + Guid.NewGuid().ToString(); + Container localContainer = await localDatabase.CreateContainerIfNotExistsAsync(uniqueContainerName, "/pk"); + + string pk = "pk_query"; + List items = this.GenerateItems(pk).ToList(); + + List itemsCreated = new List(); + foreach (TestObject item in items) + { + try + { + ItemResponse response = await localContainer.CreateItemAsync(item, new PartitionKey(item.Pk)); + if (response.StatusCode == HttpStatusCode.Created) + { + itemsCreated.Add(item); + } + } + catch (CosmosException) + { + } + } - List createdItems = await this.CreateItemsSafeAsync(items); + string query = $"SELECT * FROM c WHERE c.pk = '{pk}'"; + FeedIterator iterator = localContainer.GetItemQueryIterator(query); - string query = $"SELECT * FROM c WHERE c.pk = '{pk}'"; - FeedIterator iterator = this.container.GetItemQueryIterator(query); + int count = 0; + while (iterator.HasMoreResults) + { + FeedResponse response = await iterator.ReadNextAsync(); + count += response.Count; + } - int count = 0; - while (iterator.HasMoreResults) - { - FeedResponse response = await iterator.ReadNextAsync(); - count += response.Count; + Assert.AreEqual(itemsCreated.Count, count); } + finally + { + if (localDatabase != null) + { + try + { + await localDatabase.DeleteAsync(); + } + catch { } + } - Assert.AreEqual(createdItems.Count, count); + if (localClient != null) + { + localClient.Dispose(); + } + } } [TestMethod] @@ -927,87 +1113,247 @@ public async Task TransactionalBatchCreateItemsTest() } } - [TestMethod] - [TestCategory("ThinClient")] - public async Task RegionalFailoverWithHttpRequestException_EnsuresThinClientHeaderInRefreshRequest() - { - // Arrange - Environment.SetEnvironmentVariable(ConfigurationManager.ThinClientModeEnabled, "True"); - - bool headerFoundInRefreshRequest = false; - int accountRefreshCount = 0; - bool hasThrown = false; - - JsonSerializerOptions jsonSerializerOptions = new JsonSerializerOptions - { - PropertyNamingPolicy = null, - PropertyNameCaseInsensitive = true, - DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull - }; - CosmosSystemTextJsonSerializer serializer = new CosmosSystemTextJsonSerializer(jsonSerializerOptions); - - FaultInjectionDelegatingHandler faultHandler = new FaultInjectionDelegatingHandler( - (request) => - { - // Check for account refresh requests (GET to "/" with HTTP/1.1) - if (request.Method == HttpMethod.Get && - request.RequestUri.AbsolutePath == "/" && - request.Version == new Version(1, 1)) - { - accountRefreshCount++; - - // Only check header after we've thrown the exception - if (hasThrown) - { - if (request.Headers.TryGetValues( - ThinClientConstants.EnableThinClientEndpointDiscoveryHeaderName, - out IEnumerable headerValues)) - { - if (headerValues.Contains("True")) - { - headerFoundInRefreshRequest = true; - } - } - } - } - - // Throw HttpRequestException only ONCE on ThinClient POST requests - if (!hasThrown && - request.Method == HttpMethod.Post && - request.Version == new Version(2, 0)) - { - hasThrown = true; - throw new HttpRequestException("Simulated endpoint failure"); - } - }); - - CosmosClientBuilder builder = new CosmosClientBuilder(this.connectionString) - .WithConnectionModeGateway() - .WithCustomSerializer(serializer) - .WithHttpClientFactory(() => new HttpClient(faultHandler)); - - using CosmosClient client = builder.Build(); - - string uniqueDbName = "TestFailoverDb_" + Guid.NewGuid().ToString(); - Database database = await client.CreateDatabaseIfNotExistsAsync(uniqueDbName); - string uniqueContainerName = "TestFailoverContainer_" + Guid.NewGuid().ToString(); - Container container = await database.CreateContainerIfNotExistsAsync(uniqueContainerName, "/pk"); - - string pk = "pk_failover_test"; - TestObject testItem = this.GenerateItems(pk).First(); - - // Act - CreateItemAsync will fail once, then SDK retries and succeeds - ItemResponse response = await container.CreateItemAsync(testItem, new PartitionKey(testItem.Pk)); - - // Assert - Assert.AreEqual(HttpStatusCode.Created, response.StatusCode, "Request should succeed after retry"); - Assert.IsTrue(hasThrown, "Exception should have been thrown once"); - Assert.IsTrue(headerFoundInRefreshRequest, "Account refresh after HttpRequestException should contain thin client header"); - - // Cleanup - await database.DeleteAsync(); - } - + [TestMethod] + [TestCategory("ThinClient")] + public async Task TestThinClientQueryPlanWithOrderBy() + { + List items = new List(); + string commonPk = "pk_orderby_test_" + Guid.NewGuid().ToString(); + + try + { + Environment.SetEnvironmentVariable(ConfigurationManager.BypassQueryParsing, Boolean.TrueString); + + for (int i = 0; i < 5; i++) + { + items.Add(new TestObject + { + Id = Guid.NewGuid().ToString(), + Pk = commonPk, + Other = $"Item_{i:D3}", + }); + } + + List createdItems = await this.CreateItemsSafeAsync(items); + Assert.AreEqual(5, createdItems.Count, "All items should be created"); + + // Execute ORDER BY query - this requires QueryPlan and EPK range conversion + string query = "SELECT * FROM c WHERE c.pk = @pk ORDER BY c.other DESC"; + QueryDefinition queryDef = new QueryDefinition(query).WithParameter("@pk", commonPk); + + FeedIterator iterator = this.container.GetItemQueryIterator(queryDef); + + List results = new List(); + int pageCount = 0; + + while (iterator.HasMoreResults) + { + FeedResponse response = await iterator.ReadNextAsync(); + results.AddRange(response); + pageCount++; + + string diagnostics = response.Diagnostics.ToString(); + Assert.IsTrue(diagnostics.Contains("|F4"), $"Page {pageCount}: Should use ThinClient"); + } + + Assert.AreEqual(5, results.Count, "Should return all 5 items"); + + } + finally + { + Environment.SetEnvironmentVariable(ConfigurationManager.BypassQueryParsing, null); + + foreach (TestObject item in items) + { + try + { + await this.container.DeleteItemAsync(item.Id, new PartitionKey(item.Pk)); + } + catch { } + } + } + } + + [TestMethod] + [TestCategory("ThinClient")] + public async Task TestThinClientQueryPlanCrossPartitionWithFilter() + { + List items = new List(); + string baseGuid = Guid.NewGuid().ToString(); + + try + { + Environment.SetEnvironmentVariable(ConfigurationManager.BypassQueryParsing, "True"); + + string[] partitionKeys = { + $"pk_filter_1_{baseGuid}", + $"pk_filter_2_{baseGuid}", + $"pk_filter_3_{baseGuid}" + }; + + for (int pkIndex = 0; pkIndex < partitionKeys.Length; pkIndex++) + { + for (int i = 0; i < 3; i++) + { + items.Add(new TestObject + { + Id = Guid.NewGuid().ToString(), + Pk = partitionKeys[pkIndex], + Other = $"Value_{i}", + }); + } + } + + List createdItems = await this.CreateItemsSafeAsync(items); + Assert.AreEqual(9, createdItems.Count, "All 9 items should be created"); + + string query = "SELECT * FROM c ORDER BY c._ts"; + + FeedIterator iterator = this.container.GetItemQueryIterator(query); + + List results = new List(); + int pageCount = 0; + + while (iterator.HasMoreResults) + { + FeedResponse response = await iterator.ReadNextAsync(); + results.AddRange(response); + pageCount++; + + string diagnostics = response.Diagnostics.ToString(); + Assert.IsTrue(diagnostics.Contains("|F4"), $"Page {pageCount}: Should use ThinClient"); + } + + Assert.IsTrue(results.Count >= 9, + $"Should return at least 9 items, got {results.Count}"); + + int foundCount = 0; + foreach (TestObject item in createdItems) + { + if (results.Any(r => r.Id == item.Id)) + { + foundCount++; + } + } + + Assert.IsTrue(foundCount >= 9, + $"Should find all 9 test items in results, found {foundCount}"); + + } + finally + { + Environment.SetEnvironmentVariable(ConfigurationManager.BypassQueryParsing, null); + + foreach (TestObject item in items) + { + try + { + await this.container.DeleteItemAsync(item.Id, new PartitionKey(item.Pk)); + } + catch { } + } + } + } + + [TestMethod] + [TestCategory("ThinClient")] + public async Task TestThinClientQueryPlanMultiPartitionFanout() + { + List items = new List(); + string baseGuid = Guid.NewGuid().ToString(); + + try + { + Environment.SetEnvironmentVariable(ConfigurationManager.BypassQueryParsing, Boolean.TrueString); + + // Create items across many distinct partition keys to ensure multi-partition fanout + int partitionCount = 10; + int itemsPerPartition = 3; + + for (int pkIndex = 0; pkIndex < partitionCount; pkIndex++) + { + string pk = $"pk_fanout_{pkIndex}_{baseGuid}"; + for (int i = 0; i < itemsPerPartition; i++) + { + items.Add(new TestObject + { + Id = Guid.NewGuid().ToString(), + Pk = pk, + Other = $"Partition_{pkIndex}_Item_{i}", + }); + } + } + + int totalExpected = partitionCount * itemsPerPartition; + List createdItems = await this.CreateItemsSafeAsync(items); + Assert.AreEqual(totalExpected, createdItems.Count, $"All {totalExpected} items should be created"); + + // Execute a cross-partition ORDER BY query (requires QueryPlan + fanout) + string query = "SELECT * FROM c WHERE STARTSWITH(c.other, 'Partition_') ORDER BY c.other ASC"; + + // Run query via ThinClient mode + FeedIterator thinClientIterator = this.container.GetItemQueryIterator(query); + + List thinClientResults = new List(); + while (thinClientIterator.HasMoreResults) + { + FeedResponse response = await thinClientIterator.ReadNextAsync(); + thinClientResults.AddRange(response); + + string diagnostics = response.Diagnostics.ToString(); + Assert.IsTrue(diagnostics.Contains("|F4"), "Should use ThinClient mode"); + } + + // Verify all items are returned + int foundCount = createdItems.Count(created => + thinClientResults.Any(r => r.Id == created.Id)); + Assert.AreEqual(totalExpected, foundCount, + $"Should find all {totalExpected} test items in fanout results, found {foundCount}"); + + // Compare with Gateway mode results to verify correctness + using CosmosClient gatewayClient = new CosmosClient( + this.connectionString, + new CosmosClientOptions() + { + ConnectionMode = ConnectionMode.Gateway, + Serializer = this.cosmosSystemTextJsonSerializer, + }); + + Container gatewayContainer = gatewayClient.GetContainer(this.database.Id, this.container.Id); + FeedIterator gatewayIterator = gatewayContainer.GetItemQueryIterator(query); + + List gatewayResults = new List(); + while (gatewayIterator.HasMoreResults) + { + FeedResponse response = await gatewayIterator.ReadNextAsync(); + gatewayResults.AddRange(response); + } + + // ThinClient and Gateway should return the same item count + Assert.AreEqual(gatewayResults.Count, thinClientResults.Count, + $"ThinClient ({thinClientResults.Count}) and Gateway ({gatewayResults.Count}) should return the same number of items."); + + // Verify both results contain the same item IDs + HashSet thinClientIds = new HashSet(thinClientResults.Select(r => r.Id)); + HashSet gatewayIds = new HashSet(gatewayResults.Select(r => r.Id)); + Assert.IsTrue(thinClientIds.SetEquals(gatewayIds), + "ThinClient and Gateway should return the same set of items."); + } + finally + { + Environment.SetEnvironmentVariable(ConfigurationManager.BypassQueryParsing, null); + + foreach (TestObject item in items) + { + try + { + await this.container.DeleteItemAsync(item.Id, new PartitionKey(item.Pk)); + } + catch { } + } + } + } + /// /// DelegatingHandler that intercepts HTTP requests and can inject faults /// diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/BaselineTest/TestBaseline/SqlObjectVisitorBaselineTests.SqlFunctionCalls.xml b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/BaselineTest/TestBaseline/SqlObjectVisitorBaselineTests.SqlFunctionCalls.xml index c96170c087..c54faca13c 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/BaselineTest/TestBaseline/SqlObjectVisitorBaselineTests.SqlFunctionCalls.xml +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/BaselineTest/TestBaseline/SqlObjectVisitorBaselineTests.SqlFunctionCalls.xml @@ -1,4634 +1,4770 @@ - - - - _COMPARE_BSON_BINARYDATA - - - - - - -1302450251 - - - - - - _COMPARE_OBJECTS - - - - - - 1834671385 - - - - - - _M_EVAL_EQ - - - - - - -937798776 - - - - - - _M_EVAL_GT - - - - - - -937689624 - - - - - - _M_EVAL_GTE - - - - - - 148219533 - - - - - - _M_EVAL_IN - - - - - - -1737483201 - - - - - - _M_EVAL_LT - - - - - - -934760921 - - - - - - _M_EVAL_LTE - - - - - - 30540149 - - - - - - _M_EVAL_NEQ - - - - - - 49027154 - - - - - - _M_EVAL_NIN - - - - - - -258213395 - - - - - - _ObjectToArray - - - - - - -236268680 - - - - - - _PROXY_PROJECTION - - - - - - 610013180 - - - - - - _REGEX_MATCH - - - - - - -962904190 - - - - - - _ST_DISTANCE - - - - - - -1605852852 - - - - - - _ST_INTERSECTS - - - - - - -652359528 - - - - - - _ST_WITHIN - - - - - - 172593400 - - - - - - _TRY_ARRAY_CONTAINS - - - - - - 1151335009 - - - - - - ABS - - - - - - 948101022 - - - - - - ACOS - - - - - - 1660875890 - - - - - - ALL - - - - - - 948025334 - - - - - - ANY - - - - - - 948017723 - - - - - - ARRAY - - - - - - 1424287333 - - - - - - ARRAY_AVG - - - - - - 323897902 - - - - - - ARRAY_CONCAT - - - - - - -1615943767 - - - - - - ARRAY_CONTAINS - - - - - - -854810616 - - - - - - ARRAY_LENGTH - - - - - - -2104916661 - - - - - - ARRAY_SLICE - - - - - - -1823352982 - - - - - - ASIN - - - - - - 1734029734 - - - - - - ATAN - - - - - - 1730393998 - - - - - - ATN2 - - - - - - 1730444475 - - - - - - AVG - - - - - - 950772724 - - - - - - C_BINARY - - - - - - 1210908139 - - - - - - C_FLOAT32 - - - - - - -472685730 - - - - - - C_FLOAT64 - - - - - - -472681688 - - - - - - C_GUID - - - - - - -1807738202 - - - - - - C_INT16 - - - - - - -384520670 - - - - - - C_INT32 - - - - - - -384524651 - - - - - - C_INT64 - - - - - - -384155345 - - - - - - C_INT8 - - - - - - -1805552873 - - - - - - C_LIST - - - - - - 1591190418 - - - - - - C_LISTCONTAINS - - - - - - 1693035758 - - - - - - C_MAP - - - - - - 1391965105 - - - - - - C_MAPCONTAINS - - - - - - 893948196 - - - - - - C_MAPCONTAINSKEY - - - - - - -1848028241 - - - - - - C_MAPCONTAINSVALUE - - - - - - -1676685015 - - - - - - C_SET - - - - - - 1490943678 - - - - - - C_SETCONTAINS - - - - - - -1431746769 - - - - - - C_TUPLE - - - - - - 1511703250 - - - - - - C_UDT - - - - - - -1775923376 - - - - - - C_UINT32 - - - - - - 1491911555 - - - - - - CEILING - - - - - - 326203514 - - - - - - CONCAT - - - - - - -551258348 - - - - - - CONTAINS - - - - - - -1074519824 - - - - - - COS - - - - - - 950901151 - - - - - - COT - - - - - - 950887618 - - - - - - COUNT - - - - - - 1275248890 - - - - - - DateTimeAdd - - - - - - 1610287731 - - - - - - DateTimeBin - - - - - - -2042557462 - - - - - - DateTimeFormat - - - - - - -366443220 - - - - - - DateTimeDiff - - - - - - -1313869462 - - - - - - DateTimeFromParts - - - - - - 1715858645 - - - - - - DateTimePart - - - - - - 77619208 - - - - - - DateTimeToTicks - - - - - - -447006820 - - - - - - DateTimeToTimestamp - - - - - - 292597322 - - - - - - DAY - - - - - - 730252520 - - - - - - DEGREES - - - - - - -1010818131 - - - - - - DOCUMENTID - - - - - - -1119491048 - - - - - - ENDSWITH - - - - - - 1310545128 - - - - - - EXP - - - - - - 1188489945 - - - - - - FLOOR - - - - - - 1595010801 - - - - - - GetCurrentDateTime - - - - - - 628961137 - - - - - - GetCurrentTicks - - - - - - 767789568 - - - - - - GetCurrentTimestamp - - - - - - -108724186 - - - - - - IIF - - - - - - 1912853209 - - - - - - INDEX_OF - - - - - - -490611053 - - - - - - IS_ARRAY - - - - - - -605857869 - - - - - - IS_BOOL - - - - - - -1031205701 - - - - - - IS_DATETIME - - - - - - -1529891165 - - - - - - IS_DEFINED - - - - - - -673974177 - - - - - - IS_FINITE_NUMBER - - - - - - 1858549009 - - - - - - IS_NULL - - - - - - 1695544283 - - - - - - IS_NUMBER - - - - - - 499272998 - - - - - - IS_OBJECT - - - - - - 1190524845 - - - - - - IS_PRIMITIVE - - - - - - -25700510 - - - - - - IS_STRING - - - - - - 197369964 - - - - - - LastIndexOf - - - - - - -1618285648 - - - - - - LEFT - - - - - - 677167842 - - - - - - LENGTH - - - - - - 189371521 - - - - - - LIKE - - - - - - -535385742 - - - - - - LOG - - - - - - 951983273 - - - - - - LOG10 - - - - - - -1007856584 - - - - - - LOWER - - - - - - -1008756110 - - - - - - LTRIM - - - - - - -1298153349 - - - - - - MAX - - - - - - 951557458 - - - - - - MIN - - - - - - 951572494 - - - - - - MONTH - - - - - - -1384364250 - - - - - - ObjectToArray - - - - - - 1958364769 - - - - - - PI - - - - - - 759877112 - - - - - - POWER - - - - - - -2065410283 - - - - - - RADIANS - - - - - - 2096097273 - - - - - - RAND - - - - - - -1422123156 - - - - - - REPLACE - - - - - - 611232669 - - - - - - REPLICATE - - - - - - -1759231678 - - - - - - REVERSE - - - - - - -716026956 - - - - - - RIGHT - - - - - - 891700551 - - - - - - ROUND - - - - - - 1845634331 - - - - - - RTRIM - - - - - - -1534507744 - - - - - - SetDifference - - - - - - 1109260146 - - - - - - SetEqual - - - - - - -2058439928 - - - - - - SetIntersect - - - - - - -644124670 - - - - - - SetUnion - - - - - - 1177661016 - - - - - - SIGN - - - - - - 1835961124 - - - - - - SIN - - - - - - 951887607 - - - - - - SQRT - - - - - - 2142163143 - - - - - - SQUARE - - - - - - -67058548 - - - - - - STARTSWITH - - - - - - 820442350 - - - - - - ST_DISTANCE - - - - - - 1233749487 - - - - - - ST_INTERSECTS - - - - - - 1992127418 - - - - - - ST_ISVALID - - - - - - -488760339 - - - - - - ST_ISVALIDDETAILED - - - - - - 202371336 - - - - - - ST_WITHIN - - - - - - 1288817159 - - - - - - StringEquals - - - - - - 67992629 - - - - - - StringJoin - - - - - - 379070440 - - - - - - StringSplit - - - - - - 1939927191 - - - - - - StringToArray - - - - - - 470263213 - - - - - - StringToBoolean - - - - - - -1874642418 - - - - - - StringToNull - - - - - - -970438357 - - - - - - StringToNumber - - - - - - 1152869235 - - - - - - StringToObject - - - - - - 72615192 - - - - - - SUBSTRING - - - - - - -1893316890 - - - - - - SUM - - - - - - 1189519259 - - - - - - TAN - - - - - - 1189160248 - - - - - - TicksToDateTime - - - - - - 688550720 - - - - - - TimestampToDateTime - - - - - - -721974150 - - - - - - ToString - - - - - - -1554853341 - - - - - - TRIM - - - - - - -1513216505 - - - - - - TRUNC - - - - - - -1614924416 - - - - - - UPPER - - - - - - 2018810216 - - - - - - VectorDistance - - - - - - 341383158 - - - - - - YEAR - - - - - - 1895825496 - - - + + + + _COMPARE_BSON_BINARYDATA + + + + + + -1302450251 + + + + + + _COMPARE_OBJECTS + + + + + + 1834671385 + + + + + + _M_EVAL_EQ + + + + + + -937798776 + + + + + + _M_EVAL_GT + + + + + + -937689624 + + + + + + _M_EVAL_GTE + + + + + + 148219533 + + + + + + _M_EVAL_IN + + + + + + -1737483201 + + + + + + _M_EVAL_LT + + + + + + -934760921 + + + + + + _M_EVAL_LTE + + + + + + 30540149 + + + + + + _M_EVAL_NEQ + + + + + + 49027154 + + + + + + _M_EVAL_NIN + + + + + + -258213395 + + + + + + _ObjectToArray + + + + + + -236268680 + + + + + + _PROXY_PROJECTION + + + + + + 610013180 + + + + + + _REGEX_MATCH + + + + + + -962904190 + + + + + + _ST_DISTANCE + + + + + + -1605852852 + + + + + + _ST_INTERSECTS + + + + + + -652359528 + + + + + + _ST_WITHIN + + + + + + 172593400 + + + + + + _TRY_ARRAY_CONTAINS + + + + + + 1151335009 + + + + + + ABS + + + + + + 948101022 + + + + + + ACOS + + + + + + 1660875890 + + + + + + ALL + + + + + + 948025334 + + + + + + ANY + + + + + + 948017723 + + + + + + ARRAY + + + + + + 1424287333 + + + + + + ARRAY_AVG + + + + + + 323897902 + + + + + + ARRAY_CONCAT + + + + + + -1615943767 + + + + + + ARRAY_CONTAINS + + + + + + -854810616 + + + + + + ARRAY_LENGTH + + + + + + -2104916661 + + + + + + ARRAY_SLICE + + + + + + -1823352982 + + + + + + ASIN + + + + + + 1734029734 + + + + + + ATAN + + + + + + 1730393998 + + + + + + ATN2 + + + + + + 1730444475 + + + + + + AVG + + + + + + 950772724 + + + + + + C_BINARY + + + + + + 1210908139 + + + + + + C_FLOAT32 + + + + + + -472685730 + + + + + + C_FLOAT64 + + + + + + -472681688 + + + + + + C_GUID + + + + + + -1807738202 + + + + + + C_INT16 + + + + + + -384520670 + + + + + + C_INT32 + + + + + + -384524651 + + + + + + C_INT64 + + + + + + -384155345 + + + + + + C_INT8 + + + + + + -1805552873 + + + + + + C_LIST + + + + + + 1591190418 + + + + + + C_LISTCONTAINS + + + + + + 1693035758 + + + + + + C_MAP + + + + + + 1391965105 + + + + + + C_MAPCONTAINS + + + + + + 893948196 + + + + + + C_MAPCONTAINSKEY + + + + + + -1848028241 + + + + + + C_MAPCONTAINSVALUE + + + + + + -1676685015 + + + + + + C_SET + + + + + + 1490943678 + + + + + + C_SETCONTAINS + + + + + + -1431746769 + + + + + + C_TUPLE + + + + + + 1511703250 + + + + + + C_UDT + + + + + + -1775923376 + + + + + + C_UINT32 + + + + + + 1491911555 + + + + + + CEILING + + + + + + 326203514 + + + + + + CONCAT + + + + + + -551258348 + + + + + + CONTAINS + + + + + + -1074519824 + + + + + + COS + + + + + + 950901151 + + + + + + COT + + + + + + 950887618 + + + + + + COUNT + + + + + + 1275248890 + + + + + + DateTimeAdd + + + + + + 1610287731 + + + + + + DateTimeBin + + + + + + -2042557462 + + + + + + DateTimeFormat + + + + + + -366443220 + + + + + + DateTimeDiff + + + + + + -1313869462 + + + + + + DateTimeFromParts + + + + + + 1715858645 + + + + + + DateTimePart + + + + + + 77619208 + + + + + + DateTimeToTicks + + + + + + -447006820 + + + + + + DateTimeToTimestamp + + + + + + 292597322 + + + + + + DAY + + + + + + 730252520 + + + + + + DEGREES + + + + + + -1010818131 + + + + + + DOCUMENTID + + + + + + -1119491048 + + + + + + ENDSWITH + + + + + + 1310545128 + + + + + + EXP + + + + + + 1188489945 + + + + + + FLOOR + + + + + + 1595010801 + + + + + + GetCurrentDateTime + + + + + + 628961137 + + + + + + GetCurrentTicks + + + + + + 767789568 + + + + + + GetCurrentTimestamp + + + + + + -108724186 + + + + + + IIF + + + + + + 1912853209 + + + + + + INDEX_OF + + + + + + -490611053 + + + + + + IS_ARRAY + + + + + + -605857869 + + + + + + IS_BOOL + + + + + + -1031205701 + + + + + + IS_DATETIME + + + + + + -1529891165 + + + + + + IS_DEFINED + + + + + + -673974177 + + + + + + IS_FINITE_NUMBER + + + + + + 1858549009 + + + + + + IS_NULL + + + + + + 1695544283 + + + + + + IS_NUMBER + + + + + + 499272998 + + + + + + IS_OBJECT + + + + + + 1190524845 + + + + + + IS_PRIMITIVE + + + + + + -25700510 + + + + + + IS_STRING + + + + + + 197369964 + + + + + + LastIndexOf + + + + + + -1618285648 + + + + + + LastSubstringAfter + + + + + + 441842242 + + + + + + LastSubstringBefore + + + + + + 1887256552 + + + + + + LEFT + + + + + + 677167842 + + + + + + LENGTH + + + + + + 189371521 + + + + + + LIKE + + + + + + -535385742 + + + + + + LOG + + + + + + 951983273 + + + + + + LOG10 + + + + + + -1007856584 + + + + + + LOWER + + + + + + -1008756110 + + + + + + LTRIM + + + + + + -1298153349 + + + + + + MAX + + + + + + 951557458 + + + + + + MIN + + + + + + 951572494 + + + + + + MONTH + + + + + + -1384364250 + + + + + + ObjectToArray + + + + + + 1958364769 + + + + + + PI + + + + + + 759877112 + + + + + + POWER + + + + + + -2065410283 + + + + + + RADIANS + + + + + + 2096097273 + + + + + + RAND + + + + + + -1422123156 + + + + + + REPLACE + + + + + + 611232669 + + + + + + REPLICATE + + + + + + -1759231678 + + + + + + REVERSE + + + + + + -716026956 + + + + + + RIGHT + + + + + + 891700551 + + + + + + ROUND + + + + + + 1845634331 + + + + + + RTRIM + + + + + + -1534507744 + + + + + + SetDifference + + + + + + 1109260146 + + + + + + SetEqual + + + + + + -2058439928 + + + + + + SetIntersect + + + + + + -644124670 + + + + + + SetUnion + + + + + + 1177661016 + + + + + + SIGN + + + + + + 1835961124 + + + + + + SIN + + + + + + 951887607 + + + + + + SQRT + + + + + + 2142163143 + + + + + + SQUARE + + + + + + -67058548 + + + + + + STARTSWITH + + + + + + 820442350 + + + + + + ST_DISTANCE + + + + + + 1233749487 + + + + + + ST_INTERSECTS + + + + + + 1992127418 + + + + + + ST_ISVALID + + + + + + -488760339 + + + + + + ST_ISVALIDDETAILED + + + + + + 202371336 + + + + + + ST_WITHIN + + + + + + 1288817159 + + + + + + StringEquals + + + + + + 67992629 + + + + + + StringJoin + + + + + + 379070440 + + + + + + StringSplit + + + + + + 1939927191 + + + + + + StringToArray + + + + + + 470263213 + + + + + + StringToBoolean + + + + + + -1874642418 + + + + + + StringToNull + + + + + + -970438357 + + + + + + StringToNumber + + + + + + 1152869235 + + + + + + StringToObject + + + + + + 72615192 + + + + + + SUBSTRING + + + + + + -1893316890 + + + + + + SubstringAfter + + + + + + 1519983960 + + + + + + SubstringBefore + + + + + + 516840642 + + + + + + SUM + + + + + + 1189519259 + + + + + + TAN + + + + + + 1189160248 + + + + + + TicksToDateTime + + + + + + 688550720 + + + + + + TimestampToDateTime + + + + + + -721974150 + + + + + + ToString + + + + + + -1554853341 + + + + + + TRIM + + + + + + -1513216505 + + + + + + TRUNC + + + + + + -1614924416 + + + + + + UPPER + + + + + + 2018810216 + + + + + + VectorDistance + + + + + + 341383158 + + + + + + YEAR + + + + + + 1895825496 + + + \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GatewayStoreModelTest.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GatewayStoreModelTest.cs index ba3eaaeb81..38ba61c481 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GatewayStoreModelTest.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GatewayStoreModelTest.cs @@ -1333,7 +1333,7 @@ public async Task ThinClient_ProcessMessageAsync_WithUnsupportedOperations_Shoul .ReturnsAsync(successResponse); DocumentServiceRequest request = DocumentServiceRequest.Create( - operationType: OperationType.QueryPlan, + operationType: OperationType.ReadFeed, resourceType: ResourceType.Document, resourceId: "NH1uAJ6ANm0=", body: null, diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/SqlObjects/SqlObjectVisitorBaselineTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/SqlObjects/SqlObjectVisitorBaselineTests.cs index cba3cfb796..30928ea280 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/SqlObjects/SqlObjectVisitorBaselineTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/SqlObjects/SqlObjectVisitorBaselineTests.cs @@ -795,7 +795,19 @@ public void SqlFunctionCalls() SqlFunctionCallScalarExpression.CreateBuiltin( SqlFunctionCallScalarExpression.Identifiers.LastIndexOf, SqlLiteralScalarExpression.Create(SqlStringLiteral.Create("ABCDABCDABC")), - SqlLiteralScalarExpression.Create(SqlStringLiteral.Create("ABC")))), + SqlLiteralScalarExpression.Create(SqlStringLiteral.Create("ABC")))), + new SqlObjectVisitorInput( + SqlFunctionCallScalarExpression.Names.LastSubstringAfter, + SqlFunctionCallScalarExpression.CreateBuiltin( + SqlFunctionCallScalarExpression.Identifiers.LastSubstringAfter, + SqlLiteralScalarExpression.Create(SqlStringLiteral.Create("Hello world")), + SqlLiteralScalarExpression.Create(SqlStringLiteral.Create("o")))), + new SqlObjectVisitorInput( + SqlFunctionCallScalarExpression.Names.LastSubstringBefore, + SqlFunctionCallScalarExpression.CreateBuiltin( + SqlFunctionCallScalarExpression.Identifiers.LastSubstringBefore, + SqlLiteralScalarExpression.Create(SqlStringLiteral.Create("Hello world")), + SqlLiteralScalarExpression.Create(SqlStringLiteral.Create("o")))), new SqlObjectVisitorInput( SqlFunctionCallScalarExpression.Names.Left, SqlFunctionCallScalarExpression.CreateBuiltin( @@ -1061,7 +1073,19 @@ public void SqlFunctionCalls() SqlFunctionCallScalarExpression.Identifiers.Substring, SqlLiteralScalarExpression.Create(SqlStringLiteral.Create("Hello")), SqlLiteralScalarExpression.Create(SqlNumberLiteral.Create(1)), - SqlLiteralScalarExpression.Create(SqlNumberLiteral.Create(2)))), + SqlLiteralScalarExpression.Create(SqlNumberLiteral.Create(2)))), + new SqlObjectVisitorInput( + SqlFunctionCallScalarExpression.Names.SubstringAfter, + SqlFunctionCallScalarExpression.CreateBuiltin( + SqlFunctionCallScalarExpression.Identifiers.SubstringAfter, + SqlLiteralScalarExpression.Create(SqlStringLiteral.Create("Hello world")), + SqlLiteralScalarExpression.Create(SqlStringLiteral.Create("o")))), + new SqlObjectVisitorInput( + SqlFunctionCallScalarExpression.Names.SubstringBefore, + SqlFunctionCallScalarExpression.CreateBuiltin( + SqlFunctionCallScalarExpression.Identifiers.SubstringBefore, + SqlLiteralScalarExpression.Create(SqlStringLiteral.Create("Hello world")), + SqlLiteralScalarExpression.Create(SqlStringLiteral.Create("o")))), new SqlObjectVisitorInput( SqlFunctionCallScalarExpression.Names.Sum, SqlFunctionCallScalarExpression.CreateBuiltin( diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ThinClientStoreClientTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ThinClientStoreClientTests.cs index 7e400450a8..d97bb331ae 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ThinClientStoreClientTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ThinClientStoreClientTests.cs @@ -4,21 +4,25 @@ namespace Microsoft.Azure.Cosmos.Tests { - using System; - using System.Collections.ObjectModel; - using System.IO; - using System.Linq; + using System; + using System.Collections.Generic; + using System.Collections.ObjectModel; + using System.IO; + using System.Linq; using System.Net; - using System.Net.Http; + using System.Net.Http; using System.Text; using System.Threading; - using System.Threading.Tasks; - using Microsoft.Azure.Cosmos.Routing; - using Microsoft.Azure.Cosmos.Telemetry; - using Microsoft.Azure.Cosmos.Tracing; + using System.Threading.Tasks; + using Microsoft.Azure.Cosmos.Query.Core.QueryPlan; + using Microsoft.Azure.Cosmos.Routing; + using Microsoft.Azure.Cosmos.Telemetry; + using Microsoft.Azure.Cosmos.Tracing; using Microsoft.Azure.Documents; - using Microsoft.VisualStudio.TestTools.UnitTesting; - using Moq; + using Microsoft.Azure.Documents.Routing; + using Microsoft.VisualStudio.TestTools.UnitTesting; + using Moq; + using Newtonsoft.Json; [TestClass] public class ThinClientStoreClientTests @@ -391,6 +395,133 @@ public void Constructor_ShouldThrowArgumentNullException_WhenUserAgentContainerI StringAssert.Contains(ex.Message, "UserAgentContainer cannot be null"); } + #region ThinClientQueryPlanHelper Tests + + private static readonly PartitionKeyDefinition HashPartitionKeyDefinition = new PartitionKeyDefinition() + { + Paths = new Collection() { "/id" }, + Kind = PartitionKind.Hash, + }; + + [TestMethod] + [DynamicData(nameof(GetQueryPlanJsonTestCases), DynamicDataSourceType.Method)] + public void DeserializeQueryPlanResponse_ConsistentWithQueryPartitionProvider(string queryPlanJson, string description) + { + // Deserialize via ThinClientQueryPlanHelper (stream-based, as used in thin client mode) + PartitionedQueryExecutionInfo thinClientResult; + using (Stream stream = new MemoryStream(Encoding.UTF8.GetBytes(queryPlanJson))) + { + thinClientResult = ThinClientQueryPlanHelper.DeserializeQueryPlanResponse( + stream, + HashPartitionKeyDefinition); + } + + // Deserialize via QueryPartitionProvider (string-based, as used in gateway/service-interop mode) + QueryPartitionProvider queryPartitionProvider = new QueryPartitionProvider( + new Dictionary() { { "maxSqlQueryInputLength", 524288 } }); + + PartitionedQueryExecutionInfoInternal queryInfoInternal = + JsonConvert.DeserializeObject( + queryPlanJson, + new JsonSerializerSettings { DateParseHandling = DateParseHandling.None, MaxDepth = 64 }); + + PartitionedQueryExecutionInfo providerResult = queryPartitionProvider.ConvertPartitionedQueryExecutionInfo( + queryInfoInternal, + HashPartitionKeyDefinition); + + // Assert: Both paths must produce identical EPK ranges + Assert.AreEqual(providerResult.QueryRanges.Count, thinClientResult.QueryRanges.Count, description); + for (int i = 0; i < providerResult.QueryRanges.Count; i++) + { + Assert.AreEqual(providerResult.QueryRanges[i].Min, thinClientResult.QueryRanges[i].Min, $"{description} - range[{i}].Min"); + Assert.AreEqual(providerResult.QueryRanges[i].Max, thinClientResult.QueryRanges[i].Max, $"{description} - range[{i}].Max"); + Assert.AreEqual(providerResult.QueryRanges[i].IsMinInclusive, thinClientResult.QueryRanges[i].IsMinInclusive, $"{description} - range[{i}].IsMinInclusive"); + Assert.AreEqual(providerResult.QueryRanges[i].IsMaxInclusive, thinClientResult.QueryRanges[i].IsMaxInclusive, $"{description} - range[{i}].IsMaxInclusive"); + } + } + + private static IEnumerable GetQueryPlanJsonTestCases() + { + // Full range (cross-partition query) + yield return new object[] + { + @"{""queryInfo"":{""distinctType"":""None"",""top"":null,""offset"":null,""limit"":null,""orderBy"":[],""orderByExpressions"":[],""groupByExpressions"":[],""groupByAliases"":[],""aggregates"":[""CountIf""],""groupByAliasToAggregateType"":{},""rewrittenQuery"":""SELECT VALUE [{\""item\"": COUNTIF(c.valid)}]\nFROM c"",""hasSelectValue"":true,""dCountInfo"":null,""hasNonStreamingOrderBy"":false},""queryRanges"":[{""min"":[],""max"":""Infinity"",""isMinInclusive"":true,""isMaxInclusive"":false}]}", + "Full range with aggregate" + }; + + // Point query (single partition key) + yield return new object[] + { + @"{""queryInfo"":{""distinctType"":""None"",""top"":null,""offset"":null,""limit"":null,""orderBy"":[""Descending""],""orderByExpressions"":[],""groupByExpressions"":[],""groupByAliases"":[],""aggregates"":[],""groupByAliasToAggregateType"":{},""rewrittenQuery"":"""",""hasSelectValue"":false,""dCountInfo"":null,""hasNonStreamingOrderBy"":false},""queryRanges"":[{""min"":[""testValue""],""max"":[""testValue""],""isMinInclusive"":true,""isMaxInclusive"":true}]}", + "Point query with ORDER BY" + }; + + // HybridSearchQueryInfo + yield return new object[] + { + @"{""hybridSearchQueryInfo"":{""globalStatisticsQuery"":""SELECT COUNT(1) AS documentCount, [] AS fullTextStatistics\nFROM c"",""componentQueryInfos"":[],""componentWithoutPayloadQueryInfos"":[],""projectionQueryInfo"":null,""componentWeights"":null,""skip"":null,""take"":10,""requiresGlobalStatistics"":false},""queryRanges"":[{""min"":[],""max"":""Infinity"",""isMinInclusive"":true,""isMaxInclusive"":false}]}", + "HybridSearchQueryInfo" + }; + } + + [TestMethod] + public void DeserializeQueryPlanResponse_MultipleRanges_SortsOutput() + { + // Multiple ranges in deliberate reverse order to verify sorting + string queryPlanJson = @"{""queryInfo"":{""distinctType"":""None"",""top"":null,""offset"":null,""limit"":null,""orderBy"":[],""orderByExpressions"":[],""groupByExpressions"":[],""groupByAliases"":[],""aggregates"":[],""groupByAliasToAggregateType"":{},""rewrittenQuery"":"""",""hasSelectValue"":false,""dCountInfo"":null,""hasNonStreamingOrderBy"":false},""queryRanges"":[{""min"":[""zzz""],""max"":[""zzz""],""isMinInclusive"":true,""isMaxInclusive"":true},{""min"":[""aaa""],""max"":[""aaa""],""isMinInclusive"":true,""isMaxInclusive"":true},{""min"":[""mmm""],""max"":[""mmm""],""isMinInclusive"":true,""isMaxInclusive"":true}]}"; + + using Stream stream = new MemoryStream(Encoding.UTF8.GetBytes(queryPlanJson)); + + PartitionedQueryExecutionInfo result = ThinClientQueryPlanHelper.DeserializeQueryPlanResponse( + stream, + HashPartitionKeyDefinition); + + Assert.AreEqual(3, result.QueryRanges.Count); + for (int i = 0; i < result.QueryRanges.Count - 1; i++) + { + Assert.IsTrue( + string.Compare(result.QueryRanges[i].Min, result.QueryRanges[i + 1].Min, StringComparison.Ordinal) <= 0, + $"Ranges should be sorted: range[{i}].Min='{result.QueryRanges[i].Min}' should be <= range[{i + 1}].Min='{result.QueryRanges[i + 1].Min}'"); + } + } + + [TestMethod] + public void DeserializeQueryPlanResponse_InvalidInputs_FailsFast() + { + Assert.ThrowsException( + () => ThinClientQueryPlanHelper.DeserializeQueryPlanResponse(null, HashPartitionKeyDefinition), + "Null stream should throw"); + + using (Stream validStream = new MemoryStream(Encoding.UTF8.GetBytes("{}"))) + { + Assert.ThrowsException( + () => ThinClientQueryPlanHelper.DeserializeQueryPlanResponse(validStream, null), + "Null partitionKeyDefinition should throw"); + } + + using (Stream badJson = new MemoryStream(Encoding.UTF8.GetBytes("not valid json {{{"))) + { + try + { + ThinClientQueryPlanHelper.DeserializeQueryPlanResponse(badJson, HashPartitionKeyDefinition); + Assert.Fail("Malformed JSON should throw"); + } + catch (System.Text.Json.JsonException) + { + // Expected - System.Text.Json throws JsonException or a derived type for malformed JSON + } + } + + using (Stream nullJson = new MemoryStream(Encoding.UTF8.GetBytes("null"))) + { + Assert.ThrowsException( + () => ThinClientQueryPlanHelper.DeserializeQueryPlanResponse(nullJson, HashPartitionKeyDefinition), + "JSON null should throw FormatException"); + } + } + + #endregion + private ContainerProperties GetMockContainerProperties() { ContainerProperties containerProperties = new ContainerProperties diff --git a/changelog.md b/changelog.md index 80015ff399..9d0bc8404d 100644 --- a/changelog.md +++ b/changelog.md @@ -28,6 +28,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - [5447](https://github.com/Azure/azure-cosmos-dotnet-v3/pull/5447) Per Partition Automatic Failover: Adds Hub Region Processing Only While Routing Requests Failed with 404/1002 for single master accounts - [5582](https://github.com/Azure/azure-cosmos-dotnet-v3/pull/5582) Query: Adds ability to choose global vs local/focused statistics for FullTextScore - [5693](https://github.com/Azure/azure-cosmos-dotnet-v3/pull/5693) ThinClient Integration: Adds Enable Multiple Http2 connection on SocketsHttpHandler +- [5614](https://github.com/Azure/azure-cosmos-dotnet-v3/pull/5614) ThinClient Integration: Adds support for QueryPlan in thinclient mode - [5610](https://github.com/Azure/azure-cosmos-dotnet-v3/pull/5610) Refactors N-Region Synchronous Commit feature to use IServiceConfigurationReaderVNext interface. #### Fixed diff --git a/docs/sync_up_msdata_direct.md b/docs/sync_up_msdata_direct.md index 9c416aa65b..28d2ffb425 100644 --- a/docs/sync_up_msdata_direct.md +++ b/docs/sync_up_msdata_direct.md @@ -89,5 +89,38 @@ Once the feature branch builds successfully, it's time to submit the PR to `msda ## Sample Pull Requests to Sync-up msdata direct +- [[Internal] Direct package: Adds msdata/direct update from master](https://github.com/Azure/azure-cosmos-dotnet-v3/pull/5612) - [[Internal] Msdata/Direct: Refactors msdata branch with latest v3 and direct release](https://github.com/Azure/azure-cosmos-dotnet-v3/pull/3726) -- [[Internal] Msdata/Direct: Refactors msdata/direct branch with latest v3 master and Cosmos.Direct v3.30.4](https://github.com/Azure/azure-cosmos-dotnet-v3/pull/3776) \ No newline at end of file +- [[Internal] Msdata/Direct: Refactors msdata/direct branch with latest v3 master and Cosmos.Direct v3.30.4](https://github.com/Azure/azure-cosmos-dotnet-v3/pull/3776) + +## Automated Workflow (Recommended) + +For a faster and more reliable sync process, use the Copilot agent and helper script: + +### Option 1: Copilot Agent (AI-Assisted) + +Use the Copilot agent at [`.github/agents/msdata-direct-sync-agent.agent.md`](../.github/agents/msdata-direct-sync-agent.agent.md) for a guided, AI-assisted workflow. Start with this prompt: + +``` +Follow the msdata/direct sync agent plan in .github/agents/msdata-direct-sync-agent.agent.md + +Sync the msdata/direct branch with the latest v3 master and msdata direct codebase. +``` + +The agent handles: environment validation, branch creation, master merge, conflict resolution, msdata file sync, build validation, and PR creation. + +### Option 2: PowerShell Helper Script + +Run the helper script at [`tools/msdata-direct-sync-helper.ps1`](../tools/msdata-direct-sync-helper.ps1) for full automation: + +```powershell +# Full workflow (all phases) +.\tools\msdata-direct-sync-helper.ps1 -MsdataRepoPath "Q:\CosmosDB" + +# Run individual phases +.\tools\msdata-direct-sync-helper.ps1 -MsdataRepoPath "Q:\CosmosDB" -Phase Setup +.\tools\msdata-direct-sync-helper.ps1 -MsdataRepoPath "Q:\CosmosDB" -Phase Branch +.\tools\msdata-direct-sync-helper.ps1 -MsdataRepoPath "Q:\CosmosDB" -Phase Sync +.\tools\msdata-direct-sync-helper.ps1 -MsdataRepoPath "Q:\CosmosDB" -Phase Build +.\tools\msdata-direct-sync-helper.ps1 -MsdataRepoPath "Q:\CosmosDB" -Phase PR +``` \ No newline at end of file diff --git a/tools/msdata-direct-sync-helper.ps1 b/tools/msdata-direct-sync-helper.ps1 new file mode 100644 index 0000000000..0414f12e67 --- /dev/null +++ b/tools/msdata-direct-sync-helper.ps1 @@ -0,0 +1,651 @@ +<# +.SYNOPSIS + Helper script for syncing the msdata/direct branch with latest v3 master and msdata CosmosDB repo. + +.DESCRIPTION + Automates the mechanical parts of the msdata/direct sync workflow: + - Validates prerequisites (git, dotnet CLI, gh CLI) + - Creates feature branch with correct naming convention + - Merges master into the feature branch + - Configures and runs msdata_sync.ps1 + - Runs build validation + - Optionally creates a PR + + See .github/agents/msdata-direct-sync-agent.agent.md for the full workflow documentation. + See docs/sync_up_msdata_direct.md for background on the sync process. + +.PARAMETER MsdataRepoPath + Path to the local msdata CosmosDB repository clone. Required for the Sync phase. + +.PARAMETER Phase + Run a specific phase only. Valid values: Setup, Branch, Sync, Build, PR, All. + Default: All (runs all phases sequentially). + +.PARAMETER GitHubUsername + GitHub username for branch naming. If not provided, auto-detected via gh CLI or git config. + +.PARAMETER SkipBuild + Skip the build validation phase. Not recommended but useful for re-runs. + +.EXAMPLE + .\tools\msdata-direct-sync-helper.ps1 -MsdataRepoPath "Q:\CosmosDB" + +.EXAMPLE + .\tools\msdata-direct-sync-helper.ps1 -MsdataRepoPath "C:\repos\CosmosDB" -Phase Sync + +.EXAMPLE + .\tools\msdata-direct-sync-helper.ps1 -MsdataRepoPath "Q:\CosmosDB" -GitHubUsername "nalutripician" +#> + +[CmdletBinding()] +param( + [Parameter(Mandatory = $false)] + [string]$MsdataRepoPath, + + [Parameter(Mandatory = $false)] + [ValidateSet("Setup", "Branch", "Sync", "Build", "PR", "All")] + [string]$Phase = "All", + + [Parameter(Mandatory = $false)] + [string]$GitHubUsername, + + [Parameter(Mandatory = $false)] + [switch]$SkipBuild +) + +$ErrorActionPreference = "Stop" +$script:RepoRoot = git rev-parse --show-toplevel 2>$null +if (-not $script:RepoRoot) { + Write-Error "This script must be run from within the azure-cosmos-dotnet-v3 repository." + exit 1 +} + +$script:DateStamp = (Get-Date).ToString("MM_dd_yyyy") +$script:BranchName = $null +$script:PhaseResults = @{} + +# ============================================================================ +# Helper Functions +# ============================================================================ + +function Write-Phase { + param([string]$PhaseName, [string]$Message) + Write-Host "" + Write-Host "═══════════════════════════════════════════════════════════════" -ForegroundColor Cyan + Write-Host " Phase: $PhaseName" -ForegroundColor Cyan + Write-Host " $Message" -ForegroundColor Gray + Write-Host "═══════════════════════════════════════════════════════════════" -ForegroundColor Cyan + Write-Host "" +} + +function Write-Step { + param([int]$Number, [string]$Description) + Write-Host " [$Number] $Description" -ForegroundColor Yellow +} + +function Write-Success { + param([string]$Message) + Write-Host " ✅ $Message" -ForegroundColor Green +} + +function Write-Failure { + param([string]$Message) + Write-Host " ❌ $Message" -ForegroundColor Red +} + +function Write-Info { + param([string]$Message) + Write-Host " ℹ️ $Message" -ForegroundColor Gray +} + +function Get-GitHubUsername { + if ($GitHubUsername) { + return $GitHubUsername + } + + # Try gh CLI first + try { + $username = gh api user --jq '.login' 2>$null + if ($username) { + Write-Info "Detected GitHub username via gh CLI: $username" + return $username + } + } catch { } + + # Fall back to git config + $username = git config user.name 2>$null + if ($username) { + Write-Info "Using git config user.name: $username" + return $username + } + + Write-Failure "Could not detect GitHub username. Please provide -GitHubUsername parameter." + exit 1 +} + +# ============================================================================ +# Phase: Setup — Validate prerequisites +# ============================================================================ + +function Invoke-SetupPhase { + Write-Phase "Setup" "Validating prerequisites and environment" + + # Step 1: Check git + Write-Step 1 "Checking git..." + $gitVersion = git --version 2>$null + if ($LASTEXITCODE -ne 0) { + Write-Failure "git is not installed or not in PATH" + return $false + } + Write-Success "git: $gitVersion" + + # Step 2: Check dotnet + Write-Step 2 "Checking .NET SDK..." + $dotnetVersion = dotnet --version 2>$null + if ($LASTEXITCODE -ne 0) { + Write-Failure ".NET SDK is not installed or not in PATH" + return $false + } + Write-Success ".NET SDK: $dotnetVersion" + + # Step 3: Check gh CLI + Write-Step 3 "Checking GitHub CLI..." + $ghStatus = gh auth status 2>&1 + if ($LASTEXITCODE -ne 0) { + Write-Failure "GitHub CLI is not authenticated. Run: gh auth login --web" + return $false + } + Write-Success "GitHub CLI: authenticated" + + # Step 4: Check we're in the right repo + Write-Step 4 "Checking repository..." + $remote = git remote get-url origin 2>$null + if ($remote -notmatch "azure-cosmos-dotnet-v3") { + Write-Failure "Not in the azure-cosmos-dotnet-v3 repository. Remote: $remote" + return $false + } + Write-Success "Repository: azure-cosmos-dotnet-v3" + + # Step 5: Check msdata/direct branch exists + Write-Step 5 "Checking msdata/direct branch..." + $branches = git branch -a 2>$null + if ($branches -notmatch "msdata/direct") { + Write-Info "Fetching remote branches..." + git fetch origin msdata/direct --quiet 2>$null + } + $branches = git branch -a 2>$null + if ($branches -notmatch "msdata/direct") { + Write-Failure "msdata/direct branch not found. Check remote." + return $false + } + Write-Success "msdata/direct branch exists" + + # Step 6: Check msdata repo path (if needed for Sync phase) + if ($MsdataRepoPath) { + Write-Step 6 "Checking msdata CosmosDB repo path..." + if (-not (Test-Path $MsdataRepoPath)) { + Write-Failure "msdata repo path not found: $MsdataRepoPath" + return $false + } + Write-Success "msdata repo path: $MsdataRepoPath" + } else { + Write-Step 6 "msdata repo path not provided (required for Sync phase)" + Write-Info "Use -MsdataRepoPath parameter when running Sync phase" + } + + Write-Success "All prerequisites validated!" + $script:PhaseResults["Setup"] = "passed" + return $true +} + +# ============================================================================ +# Phase: Branch — Create feature branch and merge master +# ============================================================================ + +function Invoke-BranchPhase { + Write-Phase "Branch" "Creating feature branch and merging master" + + $username = Get-GitHubUsername + $script:BranchName = "users/$username/update_msdata_direct_$script:DateStamp" + + # Step 1: Fetch latest + Write-Step 1 "Fetching latest branches from origin..." + git fetch origin master --quiet 2>$null + git fetch origin msdata/direct --quiet 2>$null + Write-Success "Fetched latest master and msdata/direct" + + # Step 2: Check if branch already exists + Write-Step 2 "Checking for existing feature branch..." + $existingBranch = git branch -a 2>$null | Select-String $script:BranchName + if ($existingBranch) { + Write-Info "Feature branch already exists: $($script:BranchName)" + Write-Info "Checking out existing branch..." + git checkout $script:BranchName 2>$null + Write-Success "Checked out existing branch" + $script:PhaseResults["Branch"] = "passed" + return $true + } + + # Step 3: Create feature branch from msdata/direct + Write-Step 3 "Creating feature branch: $($script:BranchName)" + git checkout origin/msdata/direct -b $script:BranchName 2>$null + if ($LASTEXITCODE -ne 0) { + Write-Failure "Failed to create feature branch" + return $false + } + Write-Success "Created branch: $($script:BranchName)" + + # Step 4: Merge master + Write-Step 4 "Merging master into feature branch..." + $mergeOutput = git merge origin/master --no-edit 2>&1 + if ($LASTEXITCODE -ne 0) { + Write-Info "Merge conflicts detected. Listing conflicted files:" + $conflicts = git diff --name-only --diff-filter=U 2>$null + if ($conflicts) { + foreach ($file in $conflicts) { + Write-Host " CONFLICT: $file" -ForegroundColor Red + } + } + Write-Host "" + Write-Info "Attempting auto-resolution (accept master changes)..." + + # Try to auto-resolve by accepting master (theirs) changes + foreach ($file in $conflicts) { + git checkout --theirs $file 2>$null + git add $file 2>$null + } + + # Check if all conflicts are resolved + $remainingConflicts = git diff --name-only --diff-filter=U 2>$null + if ($remainingConflicts) { + Write-Failure "Some conflicts could not be auto-resolved:" + foreach ($file in $remainingConflicts) { + Write-Host " MANUAL: $file" -ForegroundColor Red + } + Write-Info "Please resolve remaining conflicts manually, then run: git merge --continue" + return $false + } + + git merge --continue --no-edit 2>$null + Write-Success "All conflicts resolved and merge completed" + } else { + Write-Success "Merge completed without conflicts" + } + + $script:PhaseResults["Branch"] = "passed" + return $true +} + +# ============================================================================ +# Phase: Sync — Run msdata_sync.ps1 +# ============================================================================ + +# Known msdata source directories (must match $sourceDir in msdata_sync.ps1) +$script:MsdataSourceDirs = @( + "\Product\SDK\.net\Microsoft.Azure.Cosmos.Direct\src\", + "\Product\Microsoft.Azure.Documents\Common\SharedFiles\", + "\Product\Microsoft.Azure.Documents\SharedFiles\Routing\", + "\Product\Microsoft.Azure.Documents\SharedFiles\Rntbd2\", + "\Product\Microsoft.Azure.Documents\SharedFiles\Rntbd\", + "\Product\Microsoft.Azure.Documents\SharedFiles\Rntbd\rntbdtokens\", + "\Product\SDK\.net\Microsoft.Azure.Documents.Client\LegacyXPlatform\", + "\Product\Cosmos\Core\Core.Trace\", + "\Product\Cosmos\Core\Core\Utilities\", + "\Product\Microsoft.Azure.Documents\SharedFiles\", + "\Product\Microsoft.Azure.Documents\SharedFiles\Collections\", + "\Product\Microsoft.Azure.Documents\SharedFiles\Query\", + "\Product\Microsoft.Azure.Documents\SharedFiles\Management\" +) + +# Files excluded from sync (must match $exclueList in msdata_sync.ps1) +$script:SyncExcludeList = @( + "AssemblyKeys.cs", + "BaseTransportClient.cs", + "CpuReaderBase.cs", + "LinuxCpuReader.cs", + "MemoryLoad.cs", + "MemoryLoadHistory.cs", + "UnsupportedCpuReader.cs", + "WindowsCpuReader.cs", + "msdata_sync.ps1" +) + +# Files handled separately by msdata_sync.ps1 (special-case copies) +$script:SpecialCaseFiles = @( + "TransportClient.cs", + "RMResources.Designer.cs", + "RMResources.resx" +) + +function Invoke-PostSyncVerification { + param( + [Parameter(Mandatory)] + [string]$MsdataPath, + [Parameter(Mandatory)] + [string]$DirectDir + ) + + Write-Step 4 "Verifying sync completeness — scanning msdata source directories for missing files..." + + $missingFiles = @() + $copiedFiles = @() + $rntbd2Dir = Join-Path $DirectDir "rntbd2" + + foreach ($sourceDir in $script:MsdataSourceDirs) { + $fullSourceDir = Join-Path $MsdataPath $sourceDir + if (-not (Test-Path $fullSourceDir)) { + Write-Info "Source directory not found (skipping): $sourceDir" + continue + } + + $isRntbd2 = $sourceDir -match "\\Rntbd2\\" + $targetDir = if ($isRntbd2) { $rntbd2Dir } else { $DirectDir } + + $sourceFiles = Get-ChildItem $fullSourceDir -Filter "*.cs" -File -ErrorAction SilentlyContinue + foreach ($file in $sourceFiles) { + $fileName = $file.Name + + # Skip excluded and special-case files + if ($script:SyncExcludeList -contains $fileName) { continue } + if ($script:SpecialCaseFiles -contains $fileName) { continue } + + $targetPath = Join-Path $targetDir $fileName + if (-not (Test-Path $targetPath)) { + $missingFiles += @{ Name = $fileName; Source = $file.FullName; Target = $targetDir; SourceDir = $sourceDir } + } + } + } + + if ($missingFiles.Count -eq 0) { + Write-Success "Post-sync verification passed — no missing files detected" + return $true + } + + Write-Info "Found $($missingFiles.Count) file(s) in msdata that are missing from v3 direct/:" + foreach ($missing in $missingFiles) { + Write-Host " MISSING: $($missing.Name) (from $($missing.SourceDir))" -ForegroundColor Yellow + } + + Write-Info "Auto-copying missing files..." + foreach ($missing in $missingFiles) { + try { + if (-not (Test-Path $missing.Target)) { + New-Item -ItemType Directory -Path $missing.Target -Force | Out-Null + } + Copy-Item $missing.Source -Destination $missing.Target -Force + $copiedFiles += $missing.Name + Write-Success "Copied: $($missing.Name) -> $($missing.Target)" + } catch { + Write-Failure "Failed to copy $($missing.Name): $_" + } + } + + if ($copiedFiles.Count -gt 0) { + Write-Host "" + Write-Success "Auto-copied $($copiedFiles.Count) missing file(s):" + foreach ($f in $copiedFiles) { + Write-Host " + $f" -ForegroundColor Green + } + } + + return $true +} + +function Invoke-SyncPhase { + Write-Phase "Sync" "Syncing Microsoft.Azure.Cosmos.Direct files from msdata repo" + + if (-not $MsdataRepoPath) { + Write-Failure "msdata repo path is required for Sync phase. Use -MsdataRepoPath parameter." + return $false + } + + if (-not (Test-Path $MsdataRepoPath)) { + Write-Failure "msdata repo path not found: $MsdataRepoPath" + return $false + } + + $syncScript = Join-Path $script:RepoRoot "Microsoft.Azure.Cosmos" "src" "direct" "msdata_sync.ps1" + + # Step 1: Locate sync script + Write-Step 1 "Locating msdata_sync.ps1..." + if (-not (Test-Path $syncScript)) { + Write-Failure "msdata_sync.ps1 not found at: $syncScript" + Write-Info "This file should exist after merging msdata/direct. Check the merge step." + return $false + } + Write-Success "Found: $syncScript" + + # Step 2: Configure sync script with msdata repo path + Write-Step 2 "Configuring msdata_sync.ps1 with repo path..." + $scriptContent = Get-Content $syncScript -Raw + $originalContent = $scriptContent + + # Replace the $baseDir line with user-provided path + $normalizedPath = $MsdataRepoPath.TrimEnd('\', '/') + $scriptContent = $scriptContent -replace '\$baseDir\s*=\s*"[^"]*"', "`$baseDir = `"$normalizedPath`"" + + Set-Content $syncScript -Value $scriptContent + Write-Success "Configured `$baseDir = `"$normalizedPath`"" + + # Step 3: Run sync script + Write-Step 3 "Running msdata_sync.ps1..." + $directDir = Join-Path $script:RepoRoot "Microsoft.Azure.Cosmos" "src" "direct" + Push-Location $directDir + try { + $syncOutput = & $syncScript 2>&1 + $syncOutput | ForEach-Object { Write-Host " $_" } + + # Check for errors in output + $errors = $syncOutput | Where-Object { $_ -match "Write-Error|False$" } + if ($errors) { + Write-Failure "Some files failed to sync:" + foreach ($err in $errors) { + Write-Host " ERROR: $err" -ForegroundColor Red + } + Write-Info "Please copy missing files manually from msdata repo, then re-run the Sync phase." + } else { + Write-Success "All files synced successfully" + } + } finally { + Pop-Location + } + + # Step 4: Verify sync completeness and auto-copy missing files + $verifyResult = Invoke-PostSyncVerification -MsdataPath $normalizedPath -DirectDir $directDir + if (-not $verifyResult) { + Write-Failure "Post-sync verification failed" + # Still revert the script before returning + Set-Content $syncScript -Value $originalContent + return $false + } + + # Step 5: Revert script path change (don't commit the local path) + Write-Step 5 "Reverting msdata_sync.ps1 path change..." + Set-Content $syncScript -Value $originalContent + Write-Success "Reverted msdata_sync.ps1 to original state" + + $script:PhaseResults["Sync"] = "passed" + return $true +} + +# ============================================================================ +# Phase: Build — Validate the build +# ============================================================================ + +function Invoke-BuildPhase { + Write-Phase "Build" "Building solution to validate sync" + + if ($SkipBuild) { + Write-Info "Build phase skipped (-SkipBuild flag)" + $script:PhaseResults["Build"] = "skipped" + return $true + } + + $solutionPath = Join-Path $script:RepoRoot "Microsoft.Azure.Cosmos.sln" + + # Step 1: Clean build + Write-Step 1 "Running clean build..." + Push-Location $script:RepoRoot + try { + $buildOutput = dotnet build $solutionPath -c Release 2>&1 + $exitCode = $LASTEXITCODE + + # Show last few lines of build output + $buildOutput | Select-Object -Last 10 | ForEach-Object { Write-Host " $_" } + + if ($exitCode -ne 0) { + Write-Failure "Build failed with exit code $exitCode" + Write-Info "Review build errors above and fix before proceeding." + Write-Info "Common fixes:" + Write-Info " - Missing files: copy from msdata repo" + Write-Info " - Namespace conflicts: resolve using statements" + Write-Info " - API changes: update method signatures" + return $false + } + + Write-Success "Build succeeded!" + } finally { + Pop-Location + } + + $script:PhaseResults["Build"] = "passed" + return $true +} + +# ============================================================================ +# Phase: PR — Create pull request +# ============================================================================ + +function Invoke-PRPhase { + Write-Phase "PR" "Creating pull request to msdata/direct" + + $username = Get-GitHubUsername + if (-not $script:BranchName) { + $script:BranchName = "users/$username/update_msdata_direct_$script:DateStamp" + } + + # Step 1: Stage and commit + Write-Step 1 "Staging and committing changes..." + Push-Location $script:RepoRoot + try { + git add -A 2>$null + $status = git status --porcelain 2>$null + if ($status) { + git commit -m "[Internal] Direct package: Adds msdata/direct update from master" 2>$null + if ($LASTEXITCODE -ne 0) { + Write-Failure "Commit failed" + return $false + } + Write-Success "Changes committed" + } else { + Write-Info "No new changes to commit" + } + + # Step 2: Push branch + Write-Step 2 "Pushing branch to origin..." + git push origin $script:BranchName 2>$null + if ($LASTEXITCODE -ne 0) { + Write-Failure "Push failed" + return $false + } + Write-Success "Branch pushed: $($script:BranchName)" + + # Step 3: Create PR + Write-Step 3 "Creating draft pull request..." + $prBody = @" +# Pull Request Template + +## Description + +Syncs the ``msdata/direct`` branch with: +- Latest ``master`` branch (v3 SDK changes) +- Latest ``Microsoft.Azure.Cosmos.Direct`` files from msdata CosmosDB repo + +### Changes Include +- Merged latest ``master`` branch into ``msdata/direct`` +- Updated ``Microsoft.Azure.Cosmos.Direct`` files via ``msdata_sync.ps1`` +- Resolved merge conflicts (accepted master changes) +- Build validated: ``dotnet build`` passes + +## Type of change + +- [x] New feature (non-breaking change which adds functionality) + +## Validation + +- [x] Local build passes (``dotnet build Microsoft.Azure.Cosmos.sln -c Release``) +"@ + + $prUrl = gh pr create --draft ` + --base "msdata/direct" ` + --title "[Internal] Direct package: Adds msdata/direct update from master" ` + --body $prBody ` + --reviewer "kirillg,khdang,adityasa,sboshra,FabianMeiswinkel,leminh98,neildsh" 2>&1 + + if ($LASTEXITCODE -ne 0) { + Write-Failure "PR creation failed: $prUrl" + return $false + } + + Write-Success "Draft PR created: $prUrl" + Write-Info "Monitor CI: gh pr checks " + Write-Info "Mark ready when CI passes: gh pr ready " + + } finally { + Pop-Location + } + + $script:PhaseResults["PR"] = "passed" + return $true +} + +# ============================================================================ +# Main Execution +# ============================================================================ + +Write-Host "" +Write-Host "╔═══════════════════════════════════════════════════════════════╗" -ForegroundColor Magenta +Write-Host "║ msdata/direct Branch Sync Helper ║" -ForegroundColor Magenta +Write-Host "║ Azure Cosmos DB .NET SDK v3 ║" -ForegroundColor Magenta +Write-Host "╚═══════════════════════════════════════════════════════════════╝" -ForegroundColor Magenta +Write-Host "" + +$phases = switch ($Phase) { + "Setup" { @("Setup") } + "Branch" { @("Branch") } + "Sync" { @("Sync") } + "Build" { @("Build") } + "PR" { @("PR") } + "All" { @("Setup", "Branch", "Sync", "Build", "PR") } +} + +foreach ($p in $phases) { + $result = switch ($p) { + "Setup" { Invoke-SetupPhase } + "Branch" { Invoke-BranchPhase } + "Sync" { Invoke-SyncPhase } + "Build" { Invoke-BuildPhase } + "PR" { Invoke-PRPhase } + } + + if (-not $result) { + Write-Host "" + Write-Failure "Phase '$p' failed. Fix the issue and re-run with -Phase $p" + Write-Host "" + exit 1 + } +} + +# Summary +Write-Host "" +Write-Host "═══════════════════════════════════════════════════════════════" -ForegroundColor Green +Write-Host " Summary" -ForegroundColor Green +Write-Host "═══════════════════════════════════════════════════════════════" -ForegroundColor Green +foreach ($key in $script:PhaseResults.Keys) { + $status = $script:PhaseResults[$key] + $icon = if ($status -eq "passed") { "✅" } elseif ($status -eq "skipped") { "⏭️" } else { "❌" } + Write-Host " $icon $key`: $status" +} +Write-Host "" +Write-Success "msdata/direct sync complete!" +Write-Host ""