From eb8f4c4ea4cb47ac6de6dfe92201a7a8577a9b1e Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Thu, 15 Jan 2026 18:45:47 -0800 Subject: [PATCH 01/43] first commit, will add e2e tests in the next one --- .../DurabilityProvider.cs | 117 ++++++- .../DurableTaskExtension.cs | 2 + .../HttpApiHandler.cs | 12 + .../Options/DurableTaskOptions.cs | 25 +- .../OverridableStates.cs | 114 +++---- .../TaskHubGrpcServer.cs | 289 ++++++++++++------ test/Common/DurableTaskEndToEndTests.cs | 131 +++++++- test/Common/TestHelpers.cs | 24 +- test/e2e/Tests/DedupeStatusesTests.cs | 12 + 9 files changed, 550 insertions(+), 176 deletions(-) create mode 100644 test/e2e/Tests/DedupeStatusesTests.cs diff --git a/src/WebJobs.Extensions.DurableTask/DurabilityProvider.cs b/src/WebJobs.Extensions.DurableTask/DurabilityProvider.cs index e621d0d3a..9a967a3ec 100644 --- a/src/WebJobs.Extensions.DurableTask/DurabilityProvider.cs +++ b/src/WebJobs.Extensions.DurableTask/DurabilityProvider.cs @@ -3,10 +3,12 @@ using System; using System.Collections.Generic; +using System.Linq; using System.Threading; using System.Threading.Tasks; using DurableTask.Core; using DurableTask.Core.Entities; +using DurableTask.Core.Exceptions; using DurableTask.Core.History; using DurableTask.Core.Query; using Microsoft.Azure.WebJobs.Host.Scale; @@ -106,7 +108,13 @@ public DurabilityProvider(string storageProviderName, IOrchestrationService serv /// /// Event source name (e.g. DurableTask-AzureStorage). /// - public virtual string EventSourceName { get; set; } + public virtual string EventSourceName { get; set; } + + /// + /// Gets or sets the amount of time in seconds before a creation request for an orchestration times out. + /// Default value is 180 seconds. + /// + internal int OrchestrationCreationRequestTimeoutInSeconds { get; set; } = 180; /// public int TaskOrchestrationDispatcherCount => this.GetOrchestrationService().TaskOrchestrationDispatcherCount; @@ -407,9 +415,44 @@ public Task CreateTaskOrchestrationAsync(TaskMessage creationMessage) } /// - public Task CreateTaskOrchestrationAsync(TaskMessage creationMessage, OrchestrationStatus[] dedupeStatuses) - { - return this.GetOrchestrationServiceClient().CreateTaskOrchestrationAsync(creationMessage, dedupeStatuses); + public async virtual Task CreateTaskOrchestrationAsync(TaskMessage creationMessage, OrchestrationStatus[] dedupeStatuses) + { + using var timeoutCts = new CancellationTokenSource(TimeSpan.FromSeconds(this.OrchestrationCreationRequestTimeoutInSeconds)); + await this.TerminateTaskOrchestrationWithReusableRunningStatusAndWaitAsync( + creationMessage.OrchestrationInstance.InstanceId, + dedupeStatuses, + timeoutCts.Token); + await this.GetOrchestrationServiceClient().CreateTaskOrchestrationAsync(creationMessage, dedupeStatuses); + } + + /// + /// Creates a new task orchestration instance using the specified creation message and dedupe statuses. + /// + /// The creation message for the orchestration. + /// An array of orchestration statuses used for "dedupping": + /// If an orchestration with the same instance ID already exists, and its status is in this array, then a + /// will be thrown. + /// If the array contains all of the running statuses (, , + /// and ), then only terminal statuses can be reused. + /// If at least one of these statuses is not included in the array, then if an instance with that status is found, it will first be terminated via + /// before a new orchestration is created. + /// The cancellation token. + /// A task that completes when the creation message for the task orchestration instance is enqueued. + /// Thrown if an orchestration with the same instance ID already exists and its status + /// is in . + /// Thrown if the operation is cancelled via . + public async Task CreateTaskOrchestrationAsync(TaskMessage creationMessage, OrchestrationStatus[] dedupeStatuses, CancellationToken cancellationToken) + { + using var timeoutCts = new CancellationTokenSource(TimeSpan.FromSeconds(this.OrchestrationCreationRequestTimeoutInSeconds)); + using var linkedCts = + CancellationTokenSource.CreateLinkedTokenSource( + cancellationToken, + timeoutCts.Token); + await this.TerminateTaskOrchestrationWithReusableRunningStatusAndWaitAsync( + creationMessage.OrchestrationInstance.InstanceId, + dedupeStatuses, + linkedCts.Token); + await this.GetOrchestrationServiceClient().CreateTaskOrchestrationAsync(creationMessage, dedupeStatuses); } /// @@ -609,6 +652,72 @@ public virtual bool TryGetTargetScaler( public virtual Task> StreamOrchestrationHistoryAsync(string instanceId, CancellationToken cancellationToken) { throw this.GetNotImplementedException(nameof(this.StreamOrchestrationHistoryAsync)); + } + + /// + /// If an orchestration exists with a status that is not in and has a running status (one of + /// , , or ), + /// then this method terminates the specified orchestration instance and waits until: + /// - The orchestration's status changes to , + /// - or the orchestration is deleted, + /// - or the operation is cancelled via the . + /// + /// The instance ID of the orchestration. + /// The dedupe statuses of the orchestration. + /// The cancellation token. + /// A task that completes when any of the above conditions are reached. + /// Thrown if the operation is cancelled via the . + /// Thrown if an orchestration already exists with status in . + private async Task TerminateTaskOrchestrationWithReusableRunningStatusAndWaitAsync( + string instanceId, + OrchestrationStatus[] dedupeStatuses, + CancellationToken cancellationToken) + { + var runningStatuses = new List() + { + OrchestrationStatus.Running, + OrchestrationStatus.Pending, + OrchestrationStatus.Suspended, + }; + + // At least one running status is reusable, so determine if an orchestration already exists with this status and terminate it if so + if (runningStatuses.Any(status => !dedupeStatuses.Contains(status))) + { + OrchestrationState orchestrationState = await this.GetOrchestrationStateAsync(instanceId, executionId: null); + + if (orchestrationState != null) + { + if (dedupeStatuses.Contains(orchestrationState.OrchestrationStatus)) + { + throw new OrchestrationAlreadyExistsException($"An orchestration with instance ID '{instanceId}' and status " + + $"'{orchestrationState.OrchestrationStatus}' already exists"); + } + + if (runningStatuses.Contains(orchestrationState.OrchestrationStatus)) + { + // Check for cancellation before attempting to terminate the orchestration + cancellationToken.ThrowIfCancellationRequested(); + + await this.ForceTerminateTaskOrchestrationAsync( + instanceId, + $"A new instance creation request has been issued for instance {instanceId} which currently has status " + + $"{orchestrationState.OrchestrationStatus}. Since the dedupe statuses of the creation request, " + + $"{string.Join(", ", dedupeStatuses)}, do not contain the orchestration's status, the orchestration has been " + + $"terminated and a new instance with the same instance ID will be created."); + + while (orchestrationState != null && orchestrationState.OrchestrationStatus != OrchestrationStatus.Terminated) + { + cancellationToken.ThrowIfCancellationRequested(); + await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken); + orchestrationState = await this.GetOrchestrationStateAsync(instanceId, executionId: null); + } + + // What should we do here? If dedupe statuses contains terminated, then the creation call afterwards will fail. + // Or should we throw an invalid argument exception if dedupeStatuses contains terminated but also allows for reuse of a running status? + // dedupeStatuses = dedupeStatuses.Except(new List() { OrchestrationStatus.Terminated }).ToArray(); + } + } + } } } } diff --git a/src/WebJobs.Extensions.DurableTask/DurableTaskExtension.cs b/src/WebJobs.Extensions.DurableTask/DurableTaskExtension.cs index 2955f2440..17c507333 100644 --- a/src/WebJobs.Extensions.DurableTask/DurableTaskExtension.cs +++ b/src/WebJobs.Extensions.DurableTask/DurableTaskExtension.cs @@ -183,6 +183,8 @@ public DurableTaskExtension( { this.OutOfProcProtocol = OutOfProcOrchestrationProtocol.OrchestratorShim; } + + this.defaultDurabilityProvider.OrchestrationCreationRequestTimeoutInSeconds = this.Options.OrchestrationCreationRequestTimeoutInSeconds; } internal DurableTaskOptions Options { get; } diff --git a/src/WebJobs.Extensions.DurableTask/HttpApiHandler.cs b/src/WebJobs.Extensions.DurableTask/HttpApiHandler.cs index 98ff1d4ba..1afa148f6 100644 --- a/src/WebJobs.Extensions.DurableTask/HttpApiHandler.cs +++ b/src/WebJobs.Extensions.DurableTask/HttpApiHandler.cs @@ -12,6 +12,7 @@ using System.Threading; using System.Threading.Tasks; using DurableTask.Core; +using DurableTask.Core.Exceptions; using DurableTask.Core.History; using Microsoft.AspNetCore.Routing; using Microsoft.AspNetCore.Routing.Template; @@ -946,6 +947,17 @@ await durableClient.DurabilityProvider.CreateTaskOrchestrationAsync( { return request.CreateErrorResponse(HttpStatusCode.BadRequest, "Invalid JSON content", e); } + catch (OrchestrationAlreadyExistsException e) + { + return request.CreateErrorResponse(HttpStatusCode.Conflict, e.Message); + } + catch (OperationCanceledException) + { + return request.CreateErrorResponse( + HttpStatusCode.RequestTimeout, + $"Create instance request exceeded timeout of {this.durableTaskOptions.OrchestrationCreationRequestTimeoutInSeconds} " + + $"seconds for instance ID {instanceId} while waiting for the termination of the existing instance with this instance ID."); + } } private static string GetHeaderValueFromHeaders(string header, HttpRequestHeaders headers) diff --git a/src/WebJobs.Extensions.DurableTask/Options/DurableTaskOptions.cs b/src/WebJobs.Extensions.DurableTask/Options/DurableTaskOptions.cs index 8cd324988..b7d3479a1 100644 --- a/src/WebJobs.Extensions.DurableTask/Options/DurableTaskOptions.cs +++ b/src/WebJobs.Extensions.DurableTask/Options/DurableTaskOptions.cs @@ -5,6 +5,7 @@ using System.Collections.Generic; using System.Net.Http; using DurableTask.AzureStorage.Partitioning; +using DurableTask.Core; using DurableTask.Core.Settings; using Microsoft.Azure.WebJobs.Extensions.DurableTask.Grpc; using Microsoft.Azure.WebJobs.Host; @@ -268,7 +269,22 @@ public string HubName /// Default is 100 seconds. /// This settings only applies when .NET 6 or greater is used. /// - public TimeSpan? GrpcHttpClientTimeout { get; set; } = TimeSpan.FromSeconds(100); + public TimeSpan? GrpcHttpClientTimeout { get; set; } = TimeSpan.FromSeconds(100); + + /// + /// Gets or sets the amount of time in seconds before a creation request for an orchestration times out. + /// Default value is 180 seconds. + /// + /// + /// This setting is applicable when is set to . + /// If an orchestration in a non-terminal state already exists with the instance ID passed to the creation request, then this + /// orchestration will be terminated before the new orchestration is created. This setting controls how long the extension will wait + /// for the orchestration to reach a status of before failing the creation request. + /// + /// + /// The number of seconds before a creation request for an orchestration times out. + /// + public int OrchestrationCreationRequestTimeoutInSeconds { get; set; } = 180; /// /// Gets or sets the local gRPC listener mode, controlling what version of gRPC listener is created. @@ -386,6 +402,13 @@ internal void Validate(INameResolver environmentVariableResolver) { throw new InvalidOperationException($"{nameof(this.MaxEntityOperationBatchSize)} must be a positive integer value."); } + + if (this.OrchestrationCreationRequestTimeoutInSeconds <= 0 || this.OrchestrationCreationRequestTimeoutInSeconds >= 230) + { + throw new InvalidOperationException($"{nameof(this.OrchestrationCreationRequestTimeoutInSeconds)} must be a positive integer value less than 230 seconds," + + $"which is the maximum amount of time that an HTTP triggered Function can take to respond to a request." + + $"See https://docs.azure.cn/en-us//azure-functions/functions-scale#timeout"); + } } internal bool IsDefaultHubName() diff --git a/src/WebJobs.Extensions.DurableTask/OverridableStates.cs b/src/WebJobs.Extensions.DurableTask/OverridableStates.cs index dde583d58..acef367ae 100644 --- a/src/WebJobs.Extensions.DurableTask/OverridableStates.cs +++ b/src/WebJobs.Extensions.DurableTask/OverridableStates.cs @@ -1,57 +1,57 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the MIT License. See LICENSE in the project root for license information. - -using System; -using DurableTask.Core; - -namespace Microsoft.Azure.WebJobs.Extensions.DurableTask -{ - /// - /// Represents options for different states that an existing orchestrator can be in to be able to be overwritten by - /// an attempt to start a new instance with the same instance Id. - /// - public enum OverridableStates - { - /// - /// Option to start a new orchestrator instance with an existing instnace Id when the existing - /// instance is in any state. - /// - AnyState, - - /// - /// Option to only start a new orchestrator instance with an existing instance Id when the existing - /// instance is in a terminated, failed, or completed state. - /// - NonRunningStates, - } - - /// - /// Extension methods for . - /// -#pragma warning disable SA1649 // File name should match first type name Justification: pairing extension methods with enum. - internal static class OverridableStatesExtensions -#pragma warning restore SA1649 // File name should match first type name - { - private static readonly OrchestrationStatus[] NonRunning = new OrchestrationStatus[] - { - OrchestrationStatus.Running, - OrchestrationStatus.ContinuedAsNew, - OrchestrationStatus.Pending, - OrchestrationStatus.Suspended, - }; - - /// - /// Gets the dedupe for a given . - /// - /// The overridable states. - /// An array of statuses to dedupe. - public static OrchestrationStatus[] ToDedupeStatuses(this OverridableStates states) - { - return states switch - { - OverridableStates.NonRunningStates => NonRunning, - _ => Array.Empty(), - }; - } - } -} +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See LICENSE in the project root for license information. + +using System; +using DurableTask.Core; + +namespace Microsoft.Azure.WebJobs.Extensions.DurableTask +{ + /// + /// Represents options for different states that an existing orchestrator can be in to be able to be overwritten by + /// an attempt to start a new instance with the same instance Id. + /// + public enum OverridableStates + { + /// + /// Option to start a new orchestrator instance with an existing instnace Id when the existing + /// instance is in any state. + /// + AnyState, + + /// + /// Option to only start a new orchestrator instance with an existing instance Id when the existing + /// instance is in a terminated, failed, or completed state. + /// + NonRunningStates, + } + + /// + /// Extension methods for . + /// +#pragma warning disable SA1649 // File name should match first type name Justification: pairing extension methods with enum. + internal static class OverridableStatesExtensions +#pragma warning restore SA1649 // File name should match first type name + { + private static readonly OrchestrationStatus[] NonRunning = new OrchestrationStatus[] + { + OrchestrationStatus.Running, + OrchestrationStatus.ContinuedAsNew, + OrchestrationStatus.Pending, + OrchestrationStatus.Suspended, + }; + + /// + /// Gets the dedupe for a given . + /// + /// The overridable states. + /// An array of statuses to dedupe. + public static OrchestrationStatus[] ToDedupeStatuses(this OverridableStates states) + { + return states switch + { + OverridableStates.NonRunningStates => NonRunning, + _ => Array.Empty(), + }; + } + } +} diff --git a/src/WebJobs.Extensions.DurableTask/TaskHubGrpcServer.cs b/src/WebJobs.Extensions.DurableTask/TaskHubGrpcServer.cs index acd048c20..d6cc70008 100644 --- a/src/WebJobs.Extensions.DurableTask/TaskHubGrpcServer.cs +++ b/src/WebJobs.Extensions.DurableTask/TaskHubGrpcServer.cs @@ -54,7 +54,23 @@ public override Task Hello(Empty request, ServerCallContext context) public async override Task StartInstance(P.CreateInstanceRequest request, ServerCallContext context) { try - { + { + var allStatuses = new List() + { + OrchestrationStatus.Running, + OrchestrationStatus.Pending, + OrchestrationStatus.Suspended, + OrchestrationStatus.Completed, + OrchestrationStatus.Failed, + OrchestrationStatus.Terminated, + }; + + OrchestrationStatus[] dedupeStatuses = allStatuses + .Except(request.OrchestrationIdReusePolicy.ReplaceableStatus + .Select(status => (OrchestrationStatus)status)) + .Union(this.extension.Options.OverridableExistingInstanceStates.ToDedupeStatuses()) + .ToArray(); + // Create the orchestration instance var instance = new OrchestrationInstance { @@ -78,7 +94,7 @@ public override Task Hello(Empty request, ServerCallContext context) // Create a new activity with the parent context ActivityContext.TryParse(traceParent, traceState, out ActivityContext parentActivityContext); - using Activity? scheduleOrchestrationActivity = TraceHelper.StartActivityForNewOrchestration(executionStartedEvent, parentActivityContext, request.RequestTime?.ToDateTimeOffset()); + using Activity? scheduleOrchestrationActivity = TraceHelper.StartActivityForNewOrchestration(executionStartedEvent, parentActivityContext, request.RequestTime?.ToDateTimeOffset()); // Schedule the orchestration await this.GetDurabilityProvider(context).CreateTaskOrchestrationAsync( @@ -87,7 +103,8 @@ await this.GetDurabilityProvider(context).CreateTaskOrchestrationAsync( Event = executionStartedEvent, OrchestrationInstance = instance, }, - this.GetStatusesNotToOverride()); + dedupeStatuses, + context.CancellationToken); return new P.CreateInstanceResponse { @@ -102,6 +119,15 @@ await this.GetDurabilityProvider(context).CreateTaskOrchestrationAsync( { throw new RpcException(new Status(StatusCode.AlreadyExists, $"An Orchestration instance with the ID {request.InstanceId} already exists.")); } + catch (OperationCanceledException) + { + throw new RpcException(new Status( + StatusCode.Cancelled, + context.CancellationToken.IsCancellationRequested + ? $"Create instance request cancelled for instance ID {request.InstanceId}" + : $"Create instance request exceeded timeout of {this.extension.Options.OrchestrationCreationRequestTimeoutInSeconds} seconds " + + $"for instance ID {request.InstanceId} while waiting for the termination of the existing instance with this instance ID.")); + } catch (Exception ex) { this.extension.TraceHelper.ExtensionWarningEvent( @@ -113,12 +139,6 @@ await this.GetDurabilityProvider(context).CreateTaskOrchestrationAsync( } } - private OrchestrationStatus[] GetStatusesNotToOverride() - { - OverridableStates overridableStates = this.extension.Options.OverridableExistingInstanceStates; - return overridableStates.ToDedupeStatuses(); - } - public async override Task RaiseEvent(P.RaiseEventRequest request, ServerCallContext context) { bool throwStatusExceptionsOnRaiseEvent = this.extension.Options.ThrowStatusExceptionsOnRaiseEvent ?? this.extension.DefaultDurabilityProvider.CheckStatusBeforeRaiseEvent; @@ -307,11 +327,11 @@ private OrchestrationStatus[] GetStatusesNotToOverride() public async override Task RewindInstance(P.RewindInstanceRequest request, ServerCallContext context) { - try - { + try + { #pragma warning disable CS0618 // Type or member is obsolete await this.GetClient(context).RewindAsync(request.InstanceId, request.Reason); -#pragma warning restore CS0618 // Type or member is obsolete +#pragma warning restore CS0618 // Type or member is obsolete } catch (ArgumentException ex) { @@ -481,87 +501,87 @@ private static P.GetInstanceResponse CreateGetInstanceResponse(OrchestrationStat }; } - public async override Task StreamInstanceHistory( - P.StreamInstanceHistoryRequest request, - IServerStreamWriter responseStream, - ServerCallContext context) - { - if (await this.GetClient(context).GetStatusAsync(request.InstanceId, showInput: false) is null) - { - throw new RpcException(new Status(StatusCode.NotFound, $"Orchestration instance with ID {request.InstanceId} was not found.")); - } - - async Task<(P.HistoryChunk, int)> AddToHistoryChunkAndStream(HistoryEvent historyEvent, P.HistoryChunk historyChunk, int currentChunkSizeInBytes) - { - P.HistoryEvent result = ProtobufUtils.ToHistoryEventProto(historyEvent); - - int currentEventSize = result.CalculateSize(); - if (currentChunkSizeInBytes + currentEventSize > MaxHistoryChunkSizeInBytes) - { - // If we exceeded the chunk size threshold, send what we have so far. - await responseStream.WriteAsync(historyChunk); - historyChunk = new (); - currentChunkSizeInBytes = 0; - } - - historyChunk.Events.Add(result); - currentChunkSizeInBytes += currentEventSize; - return (historyChunk, currentChunkSizeInBytes); - } - - try - { - int currentChunkSizeInBytes = 0; - P.HistoryChunk historyChunk = new (); - - // First, try to use the streaming API if it's implemented. - try - { - IAsyncEnumerable historyEvents = await this.GetDurabilityProvider(context).StreamOrchestrationHistoryAsync( - request.InstanceId, - context.CancellationToken); - - await foreach (HistoryEvent historyEvent in historyEvents) - { - (historyChunk, currentChunkSizeInBytes) = await AddToHistoryChunkAndStream(historyEvent, historyChunk, currentChunkSizeInBytes); - } - } - - // Otherwise default to the older non-streaming implementation. - catch (NotImplementedException) - { - string jsonHistory = await this.GetDurabilityProvider(context).GetOrchestrationHistoryAsync( - request.InstanceId, - executionId: null); - - List? historyEvents = JsonConvert.DeserializeObject>( - jsonHistory, - new JsonSerializerSettings() - { - Converters = { new HistoryEventJsonConverter() }, - }) - ?? throw new Exception($"Failed to deserialize orchestration history."); - - foreach (HistoryEvent historyEvent in historyEvents) - { - (historyChunk, currentChunkSizeInBytes) = await AddToHistoryChunkAndStream(historyEvent, historyChunk, currentChunkSizeInBytes); - } - } - - // Send the last chunk, which may be smaller than the maximum chunk size. - if (historyChunk.Events.Count > 0) - { - await responseStream.WriteAsync(historyChunk); - } - } - catch (OperationCanceledException) - { - throw new RpcException(new Status(StatusCode.Cancelled, $"Orchestration history streaming cancelled for instance {request.InstanceId}")); - } - catch (Exception ex) - { - throw new RpcException(new Status(StatusCode.Internal, $"Failed to stream orchestration history for instance {request.InstanceId}: {ex.Message}")); - } + public async override Task StreamInstanceHistory( + P.StreamInstanceHistoryRequest request, + IServerStreamWriter responseStream, + ServerCallContext context) + { + if (await this.GetClient(context).GetStatusAsync(request.InstanceId, showInput: false) is null) + { + throw new RpcException(new Status(StatusCode.NotFound, $"Orchestration instance with ID {request.InstanceId} was not found.")); + } + + async Task<(P.HistoryChunk, int)> AddToHistoryChunkAndStream(HistoryEvent historyEvent, P.HistoryChunk historyChunk, int currentChunkSizeInBytes) + { + P.HistoryEvent result = ProtobufUtils.ToHistoryEventProto(historyEvent); + + int currentEventSize = result.CalculateSize(); + if (currentChunkSizeInBytes + currentEventSize > MaxHistoryChunkSizeInBytes) + { + // If we exceeded the chunk size threshold, send what we have so far. + await responseStream.WriteAsync(historyChunk); + historyChunk = new (); + currentChunkSizeInBytes = 0; + } + + historyChunk.Events.Add(result); + currentChunkSizeInBytes += currentEventSize; + return (historyChunk, currentChunkSizeInBytes); + } + + try + { + int currentChunkSizeInBytes = 0; + P.HistoryChunk historyChunk = new (); + + // First, try to use the streaming API if it's implemented. + try + { + IAsyncEnumerable historyEvents = await this.GetDurabilityProvider(context).StreamOrchestrationHistoryAsync( + request.InstanceId, + context.CancellationToken); + + await foreach (HistoryEvent historyEvent in historyEvents) + { + (historyChunk, currentChunkSizeInBytes) = await AddToHistoryChunkAndStream(historyEvent, historyChunk, currentChunkSizeInBytes); + } + } + + // Otherwise default to the older non-streaming implementation. + catch (NotImplementedException) + { + string jsonHistory = await this.GetDurabilityProvider(context).GetOrchestrationHistoryAsync( + request.InstanceId, + executionId: null); + + List? historyEvents = JsonConvert.DeserializeObject>( + jsonHistory, + new JsonSerializerSettings() + { + Converters = { new HistoryEventJsonConverter() }, + }) + ?? throw new Exception($"Failed to deserialize orchestration history."); + + foreach (HistoryEvent historyEvent in historyEvents) + { + (historyChunk, currentChunkSizeInBytes) = await AddToHistoryChunkAndStream(historyEvent, historyChunk, currentChunkSizeInBytes); + } + } + + // Send the last chunk, which may be smaller than the maximum chunk size. + if (historyChunk.Events.Count > 0) + { + await responseStream.WriteAsync(historyChunk); + } + } + catch (OperationCanceledException) + { + throw new RpcException(new Status(StatusCode.Cancelled, $"Orchestration history streaming cancelled for instance {request.InstanceId}")); + } + catch (Exception ex) + { + throw new RpcException(new Status(StatusCode.Internal, $"Failed to stream orchestration history for instance {request.InstanceId}: {ex.Message}")); + } } private static P.TaskFailureDetails? GetFailureDetails(FailureDetails? failureDetails) @@ -620,5 +640,88 @@ private P.EntityMetadata ConvertEntityMetadata(EntityBackendQueries.EntityMetada SerializedState = metaData.SerializedState, }; } + + private async Task TerminateExistingNonTerminalInstance(P.CreateInstanceRequest request, ServerCallContext context, IEnumerable dedupeStatuses) + { + var runningStatuses = new List() + { + P.OrchestrationStatus.Running, + P.OrchestrationStatus.Pending, + P.OrchestrationStatus.Suspended, + }; + + // Only check if an existing instance with this instance ID is running if: + // 1. The user explicitly specified in their options settings that any existing instance state is overridable + // (as opposed to only terminal states), and + // 2. The reusable statuses passed to the creation request contain at least one non-terminal status. + if (this.extension.Options.OverridableExistingInstanceStates == OverridableStates.AnyState + && request.OrchestrationIdReusePolicy.ReplaceableStatus.Any(status => runningStatuses.Contains(status))) + { + OrchestrationState orchestrationState = await this.GetDurabilityProvider(context) + .GetOrchestrationStateAsync(request.InstanceId, executionId: null); + + if (orchestrationState?.OrchestrationInstance != null) + { + // If an existing instance is found, check if its status is in the list of reusable statuses. + if (!request.OrchestrationIdReusePolicy.ReplaceableStatus.Contains((P.OrchestrationStatus)orchestrationState.OrchestrationStatus)) + { + throw new OrchestrationAlreadyExistsException(); + } + + // If the existing instance is in a non-terminal state, terminate it before creating a new instance. + if (runningStatuses.Contains((P.OrchestrationStatus)orchestrationState.OrchestrationStatus)) + { + OrchestrationStatus originalStatus = orchestrationState.OrchestrationStatus; + + // Check for cancellation before attempting to terminate + if (context.CancellationToken.IsCancellationRequested) + { + throw new RpcException(new Status(StatusCode.Cancelled, $"Create instance request cancelled for instance ID {request.InstanceId}")); + } + + await this.GetClient(context).TerminateAsync(request.InstanceId, $"A new instance creation request has been issued for instance " + + $"{request.InstanceId}, which currently has status {orchestrationState.OrchestrationStatus}. Since the " + + $"{nameof(DurableTaskOptions.OverridableExistingInstanceStates)} is set to {nameof(OverridableStates.AnyState)}, and the dedupe " + + $"statuses of the creation request, {dedupeStatuses}, do not contain the orchestration's status, the orchestration has been " + + $"terminated and a new instance with the same instance ID will be created."); + + var timeoutCts = new CancellationTokenSource(TimeSpan.FromSeconds(90)); + using var linkedCts = + CancellationTokenSource.CreateLinkedTokenSource( + context.CancellationToken, + timeoutCts.Token); + + while (!linkedCts.IsCancellationRequested + && orchestrationState != null + && orchestrationState.OrchestrationStatus != OrchestrationStatus.Terminated) + { + await Task.Delay(TimeSpan.FromSeconds(1), context.CancellationToken); + orchestrationState = await this.GetDurabilityProvider(context) + .GetOrchestrationStateAsync(request.InstanceId, executionId: null); + } + + if (linkedCts.IsCancellationRequested) + { + throw new RpcException(new Status( + StatusCode.Cancelled, + context.CancellationToken.IsCancellationRequested + ? $"Create instance request cancelled for instance ID {request.InstanceId}" + : $"Create instance request exceeded timeout of 100 seconds for instance ID {request.InstanceId} " + + $"while waiting for the termination of the existing instance with this instance ID.")); + } + + this.extension.TraceHelper.ExtensionInformationalEvent( + this.extension.Options.HubName, + functionName: request.Name, + instanceId: request.InstanceId, + message: $"Successfully terminated existing instance with instance ID {request.InstanceId} and status {originalStatus} " + + $"in create instance request. Will proceed to create a new instance with this ID.", + writeToUserLogs: true); + + // What should we do if the dedupe statuses contain the Terminated status? Then the creation request after this point will fail. Should we remove this status? + } + } + } + } } } diff --git a/test/Common/DurableTaskEndToEndTests.cs b/test/Common/DurableTaskEndToEndTests.cs index 0b283869e..14cae0450 100644 --- a/test/Common/DurableTaskEndToEndTests.cs +++ b/test/Common/DurableTaskEndToEndTests.cs @@ -15,6 +15,7 @@ using System.Threading.Tasks; using Azure.Storage.Blobs; using DurableTask.Core; +using DurableTask.Core.Exceptions; using Microsoft.Azure.WebJobs.Extensions.DurableTask.ContextImplementations; using Microsoft.Azure.WebJobs.Extensions.DurableTask.Options; using Microsoft.Azure.WebJobs.Host; @@ -5274,19 +5275,39 @@ await Assert.ThrowsAsync(async () => } } + public static IEnumerable GetBooleanAndFullFeaturedStorageProviderOptionsWithOverridableStatesAndSuspend() + { + foreach (object[] data in GetBooleanAndFullFeaturedStorageProviderOptionsWithOverridableStates()) + { + yield return new object[] { data[0], data[1], data[2], true }; + yield return new object[] { data[0], data[1], data[2], false }; + } + } + + public static IEnumerable GetBooleanAndFullFeaturedStorageProviderOptionsWithOverridableStates() + { + foreach (object[] data in TestDataGenerator.GetBooleanAndFullFeaturedStorageProviderOptions()) + { + yield return new object[] { data[0], data[1], true }; + yield return new object[] { data[0], data[1], false }; + } + } + [Theory] [Trait("Category", PlatformSpecificHelpers.TestCategory)] - [MemberData(nameof(TestDataGenerator.GetBooleanAndFullFeaturedStorageProviderOptions), MemberType = typeof(TestDataGenerator))] - public async Task DedupeStates_AnyState(bool extendedSessions, string storageProvider) + [MemberData(nameof(GetBooleanAndFullFeaturedStorageProviderOptionsWithOverridableStatesAndSuspend))] + public async Task OverridableStates_RunningStatusesCorrectlyDeduped(bool extendedSessions, string storageProvider, bool anyStateOverridable, bool suspend) { - DurableTaskOptions options = new DurableTaskOptions(); - options.OverridableExistingInstanceStates = OverridableStates.AnyState; + DurableTaskOptions options = new () + { + OverridableExistingInstanceStates = anyStateOverridable ? OverridableStates.AnyState : OverridableStates.NonRunningStates, + }; var instanceId = "OverridableStatesAnyStateTest_" + Guid.NewGuid().ToString("N"); using (ITestHost host = TestHelpers.GetJobHost( this.loggerProvider, - nameof(this.DedupeStates_AnyState), + nameof(this.OverridableStates_RunningStatusesCorrectlyDeduped), extendedSessions, storageProviderType: storageProvider, options: options)) @@ -5308,16 +5329,108 @@ public async Task DedupeStates_AnyState(bool extendedSessions, string storagePro // Make sure it's still running and didn't complete early (or fail). var status = await client.GetStatusAsync(); - Assert.True( - status?.RuntimeStatus == OrchestrationRuntimeStatus.Running || - status?.RuntimeStatus == OrchestrationRuntimeStatus.ContinuedAsNew); + Assert.Equal(OrchestrationRuntimeStatus.Running, status?.RuntimeStatus); - await host.StartOrchestratorAsync(nameof(TestOrchestrations.Counter), initialValue, this.output, instanceId: instanceId); + if (suspend) + { + await client.SuspendAsync("suspend for test"); + DurableOrchestrationStatus suspendedStatus = await client.WaitForStatusChange(this.output, OrchestrationRuntimeStatus.Suspended); + Assert.Equal(OrchestrationRuntimeStatus.Suspended, suspendedStatus?.RuntimeStatus); + } + + FunctionInvocationException invocationException = null; + try + { + await host.StartOrchestratorAsync(nameof(TestOrchestrations.Counter), initialValue, this.output, instanceId: instanceId); + } + catch (FunctionInvocationException caughtException) + { + invocationException = caughtException; + } await host.StopAsync(); + + // If any state is reusable, confirm that there is evidence the existing orchestration was terminated before the new one was created + if (anyStateOverridable && this.useTestLogger) + { + IReadOnlyCollection durableTaskCoreLogs = this.loggerProvider.CreatedLoggers.Single(l => l.Category == "DurableTask.Core").LogMessages; + Assert.Contains(durableTaskCoreLogs, log => log.ToString().StartsWith($"{instanceId}: Orchestration completed with a 'Terminated' status")); + } + + // Otherwise confirm that an exception was thrown when trying to create a new orchestration when one with a nonterminal status already exists + else if (!anyStateOverridable) + { + Assert.NotNull(invocationException); + Assert.NotNull(invocationException.InnerException); + Assert.IsType(invocationException.InnerException); + } } } + [Theory] + [Trait("Category", PlatformSpecificHelpers.TestCategory)] + [MemberData(nameof(GetBooleanAndFullFeaturedStorageProviderOptionsWithOverridableStates))] + public async Task OverridableStates_TerminalStatusesAlwaysReusable(bool extendedSessions, string storageProvider, bool anyStateOverridable) + { + DurableTaskOptions options = new () + { + OverridableExistingInstanceStates = anyStateOverridable ? OverridableStates.AnyState : OverridableStates.NonRunningStates, + }; + + string instanceIdBase = "OverridableStatesTerminalTest_" + Guid.NewGuid().ToString("N"); + + using ITestHost host = TestHelpers.GetJobHost( + this.loggerProvider, + nameof(this.OverridableStates_TerminalStatusesAlwaysReusable), + extendedSessions, + storageProviderType: storageProvider, + options: options); + await host.StartAsync(); + + int initialValue = 0; + + // Test for all terminal statuses: Completed, Failed, Terminated + foreach (OrchestrationRuntimeStatus terminalStatus in new[] { OrchestrationRuntimeStatus.Completed, OrchestrationRuntimeStatus.Failed, OrchestrationRuntimeStatus.Terminated }) + { + string instanceId = instanceIdBase + "_" + terminalStatus; + + TestDurableClient client; + if (terminalStatus != OrchestrationRuntimeStatus.Failed) + { + client = await host.StartOrchestratorAsync(nameof(TestOrchestrations.Counter), initialValue, this.output, instanceId: instanceId); + } + else + { + client = await host.StartOrchestratorAsync(nameof(TestOrchestrations.ThrowOrchestrator), string.Empty, this.output, instanceId: instanceId); + } + + await client.WaitForStartupAsync(this.output); + DurableOrchestrationStatus status = null; + + if (terminalStatus == OrchestrationRuntimeStatus.Completed) + { + await client.RaiseEventAsync("operation", "end", this.output); + } + else if (terminalStatus == OrchestrationRuntimeStatus.Terminated) + { + await client.TerminateAsync("test terminate"); + } + + status = await client.WaitForCompletionAsync(this.output); + Assert.NotNull(status); + Assert.Equal(terminalStatus, status.RuntimeStatus); + + // Should always be able to start a new orchestration with the same instanceId + await host.StartOrchestratorAsync( + terminalStatus == OrchestrationRuntimeStatus.Failed ? nameof(TestOrchestrations.ThrowOrchestrator) : nameof(TestOrchestrations.Counter), + terminalStatus == OrchestrationRuntimeStatus.Failed ? string.Empty : initialValue, + this.output, + instanceId: instanceId); + } + + await host.StopAsync(); + } + [Fact] [Trait("Category", PlatformSpecificHelpers.TestCategory)] public async Task CallActivity_Like_From_Azure_Portal() diff --git a/test/Common/TestHelpers.cs b/test/Common/TestHelpers.cs index 3a85dc875..e407a4db4 100644 --- a/test/Common/TestHelpers.cs +++ b/test/Common/TestHelpers.cs @@ -412,15 +412,15 @@ public static Task DeleteTaskHubResources(string testName, bool enableExtendedSe var service = new AzureStorageOrchestrationService(settings); return service.DeleteAsync(); - } - - public static void AssertLogMessageSequence( - ITestOutputHelper testOutput, - TestLoggerProvider loggerProvider, - string testName, - string instanceId, - bool filterOutReplayLogs, - string[] orchestratorFunctionNames, + } + + public static void AssertLogMessageSequence( + ITestOutputHelper testOutput, + TestLoggerProvider loggerProvider, + string testName, + string instanceId, + bool filterOutReplayLogs, + string[] orchestratorFunctionNames, string activityFunctionName = null) { List messageIds; @@ -488,9 +488,9 @@ private static List GetLogMessages( { // It is assumed that the 5th log message is a sub-orchestration instanceIds.Add(GetInstanceId(logMessages[4].FormattedMessage)); - } - - Assert.True( + } + + Assert.True( logMessages.TrueForAll(m => m.Category.Equals(LogCategory, StringComparison.InvariantCultureIgnoreCase))); return logMessages; diff --git a/test/e2e/Tests/DedupeStatusesTests.cs b/test/e2e/Tests/DedupeStatusesTests.cs new file mode 100644 index 000000000..5c62ece88 --- /dev/null +++ b/test/e2e/Tests/DedupeStatusesTests.cs @@ -0,0 +1,12 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Microsoft.Azure.Durable.Tests.DotnetIsolatedE2E +{ + internal class DedupeStatusesTests + { + } +} From 28bd9937f7536b55460b3324f8a629073cabdd22 Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Fri, 16 Jan 2026 21:44:51 -0800 Subject: [PATCH 02/43] added the e2e tests: --- .../DurabilityProvider.cs | 1 + .../TaskHubGrpcServer.cs | 10 +- .../Apps/BasicDotNetIsolated/HelloCities.cs | 73 +++++++- test/e2e/Apps/BasicDotNetIsolated/host.json | 3 +- test/e2e/Apps/BasicJava/host.json | 3 +- .../main/java/com/function/HelloCities.java | 4 +- test/e2e/Apps/BasicNode/host.json | 3 +- .../BasicNode/src/functions/HelloCities.ts | 6 +- .../StartOrchestration/run.ps1 | 3 +- test/e2e/Apps/BasicPowerShell/host.json | 3 +- test/e2e/Apps/BasicPython/hello_cities.py | 4 +- test/e2e/Apps/BasicPython/host.json | 3 +- test/e2e/Tests/DedupeStatusesTests.cs | 12 -- test/e2e/Tests/Tests/DedupeStatusesTests.cs | 174 ++++++++++++++++++ .../Tests/DistributedTracingEntitiesTests.cs | 4 +- test/e2e/Tests/Tests/PurgeInstancesTests.cs | 27 +-- 16 files changed, 290 insertions(+), 43 deletions(-) delete mode 100644 test/e2e/Tests/DedupeStatusesTests.cs create mode 100644 test/e2e/Tests/Tests/DedupeStatusesTests.cs diff --git a/src/WebJobs.Extensions.DurableTask/DurabilityProvider.cs b/src/WebJobs.Extensions.DurableTask/DurabilityProvider.cs index 9a967a3ec..2fe264db2 100644 --- a/src/WebJobs.Extensions.DurableTask/DurabilityProvider.cs +++ b/src/WebJobs.Extensions.DurableTask/DurabilityProvider.cs @@ -415,6 +415,7 @@ public Task CreateTaskOrchestrationAsync(TaskMessage creationMessage) } /// + /// should this method comment be updated too? to mention terminating existing instances? public async virtual Task CreateTaskOrchestrationAsync(TaskMessage creationMessage, OrchestrationStatus[] dedupeStatuses) { using var timeoutCts = new CancellationTokenSource(TimeSpan.FromSeconds(this.OrchestrationCreationRequestTimeoutInSeconds)); diff --git a/src/WebJobs.Extensions.DurableTask/TaskHubGrpcServer.cs b/src/WebJobs.Extensions.DurableTask/TaskHubGrpcServer.cs index d641bcae9..14b8f1c2f 100644 --- a/src/WebJobs.Extensions.DurableTask/TaskHubGrpcServer.cs +++ b/src/WebJobs.Extensions.DurableTask/TaskHubGrpcServer.cs @@ -63,10 +63,16 @@ public override Task Hello(Empty request, ServerCallContext context) OrchestrationStatus.Failed, OrchestrationStatus.Terminated, }; + + // Not all clients are necessarily configured to set the OrchestrationIdReusePolicy field of the request. + // If it is null, we assume that they do not support per-request-dedupe statuses, and default to using just + // the OverridableExistingInstanceStates setting instead. + List reusableStatuses = request.OrchestrationIdReusePolicy is null + ? allStatuses + : request.OrchestrationIdReusePolicy.ReplaceableStatus.Select(status => (OrchestrationStatus)status).ToList(); OrchestrationStatus[] dedupeStatuses = allStatuses - .Except(request.OrchestrationIdReusePolicy.ReplaceableStatus - .Select(status => (OrchestrationStatus)status)) + .Except(reusableStatuses) .Union(this.extension.Options.OverridableExistingInstanceStates.ToDedupeStatuses()) .ToArray(); diff --git a/test/e2e/Apps/BasicDotNetIsolated/HelloCities.cs b/test/e2e/Apps/BasicDotNetIsolated/HelloCities.cs index 70b490646..ced2caa5f 100644 --- a/test/e2e/Apps/BasicDotNetIsolated/HelloCities.cs +++ b/test/e2e/Apps/BasicDotNetIsolated/HelloCities.cs @@ -1,6 +1,8 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the MIT License. See License.txt in the project root for license information. +using System.Net; +using DurableTask.Core.Exceptions; using Microsoft.Azure.Functions.Worker; using Microsoft.Azure.Functions.Worker.Http; using Microsoft.DurableTask; @@ -41,12 +43,20 @@ public static async Task StartOrchestration( [HttpTrigger(AuthorizationLevel.Anonymous, "get", "post")] HttpRequestData req, [DurableClient] DurableTaskClient client, FunctionContext executionContext, - string orchestrationName) + string orchestrationName, + string? instanceId) { ILogger logger = executionContext.GetLogger(nameof(StartOrchestration)); - // Function input comes from the request content. - string instanceId = await client.ScheduleNewOrchestrationInstanceAsync(orchestrationName); + // Function input comes from the request content. + if (instanceId is not null) + { + await client.ScheduleNewOrchestrationInstanceAsync(orchestrationName, new StartOrchestrationOptions(InstanceId: instanceId)); + } + else + { + instanceId = await client.ScheduleNewOrchestrationInstanceAsync(orchestrationName); + } logger.LogInformation("Started orchestration with ID = '{instanceId}'.", instanceId); @@ -60,18 +70,71 @@ public static async Task HttpStartScheduled( [HttpTrigger(AuthorizationLevel.Anonymous, "get", "post")] HttpRequestData req, [DurableClient] DurableTaskClient client, FunctionContext executionContext, - DateTime scheduledStartTime) + DateTime scheduledStartTime, + string? instanceId) { ILogger logger = executionContext.GetLogger("HelloCities_HttpStart"); var startOptions = new StartOrchestrationOptions(StartAt: scheduledStartTime); // Function input comes from the request content. - string instanceId = await client.ScheduleNewOrchestrationInstanceAsync( + if (instanceId is not null) + { + startOptions = startOptions with { InstanceId = instanceId }; + } + instanceId = await client.ScheduleNewOrchestrationInstanceAsync( nameof(HelloCities), startOptions); logger.LogInformation("Started orchestration with ID = '{instanceId}'.", instanceId); + // Returns an HTTP 202 response with an instance management payload. + // See https://learn.microsoft.com/azure/azure-functions/durable/durable-functions-http-api#start-orchestration + return await client.CreateCheckStatusResponseAsync(req, instanceId); + } + + [Function(nameof(StartOrchestration_DedupeStatuses))] + public static async Task StartOrchestration_DedupeStatuses( + [HttpTrigger(AuthorizationLevel.Anonymous, "get", "post")] HttpRequestData req, + [DurableClient] DurableTaskClient client, + FunctionContext executionContext, + string orchestrationName, + string instanceId, + string[] dedupeStatuses, + DateTime? scheduledStartTime) + { + ILogger logger = executionContext.GetLogger(nameof(StartOrchestration_DedupeStatuses)); + + StartOrchestrationOptions startOptions = new(InstanceId: instanceId); + + var parsedStatuses = new OrchestrationRuntimeStatus[dedupeStatuses.Length]; + for (int i = 0; i < dedupeStatuses.Length; i++) + { + string statusStr = dedupeStatuses[i]; + if (!Enum.TryParse(statusStr, ignoreCase: true, out var status)) + { + throw new ArgumentException($"Invalid OrchestrationRuntimeStatus value: '{statusStr}'", nameof(dedupeStatuses)); + } + parsedStatuses[i] = status; + } + startOptions = startOptions.WithDedupeStatuses(parsedStatuses); + + if (scheduledStartTime is not null) + { + startOptions = startOptions with { StartAt = scheduledStartTime }; + } + + // Function input comes from the request content. + try + { + await client.ScheduleNewOrchestrationInstanceAsync(orchestrationName, startOptions); + } + catch (Exception) + { + return req.CreateResponse(HttpStatusCode.BadRequest); + } + + logger.LogInformation("Started orchestration with ID = '{instanceId}'.", instanceId); + // Returns an HTTP 202 response with an instance management payload. // See https://learn.microsoft.com/azure/azure-functions/durable/durable-functions-http-api#start-orchestration return await client.CreateCheckStatusResponseAsync(req, instanceId); diff --git a/test/e2e/Apps/BasicDotNetIsolated/host.json b/test/e2e/Apps/BasicDotNetIsolated/host.json index 036495c64..58c3e7e5b 100644 --- a/test/e2e/Apps/BasicDotNetIsolated/host.json +++ b/test/e2e/Apps/BasicDotNetIsolated/host.json @@ -11,7 +11,8 @@ "versionMatchStrategy": "CurrentOrOlder", "versionFailureStrategy": "Fail", "extendedSessionsEnabled": true, - "extendedSessionIdleTimeoutInSeconds": 30 + "extendedSessionIdleTimeoutInSeconds": 30, + "overridableExistingInstanceStates": "AnyState" } } } \ No newline at end of file diff --git a/test/e2e/Apps/BasicJava/host.json b/test/e2e/Apps/BasicJava/host.json index de73a2331..23bbaaa21 100644 --- a/test/e2e/Apps/BasicJava/host.json +++ b/test/e2e/Apps/BasicJava/host.json @@ -9,7 +9,8 @@ }, "defaultVersion": "2.0", "versionMatchStrategy": "CurrentOrOlder", - "versionFailureStrategy": "Fail" + "versionFailureStrategy": "Fail", + "overridableExistingInstanceStates": "AnyState" } } } diff --git a/test/e2e/Apps/BasicJava/src/main/java/com/function/HelloCities.java b/test/e2e/Apps/BasicJava/src/main/java/com/function/HelloCities.java index cf46da1ef..133a3f192 100644 --- a/test/e2e/Apps/BasicJava/src/main/java/com/function/HelloCities.java +++ b/test/e2e/Apps/BasicJava/src/main/java/com/function/HelloCities.java @@ -52,7 +52,8 @@ public HttpResponseMessage startOrchestration( final ExecutionContext context) { DurableTaskClient client = durableContext.getClient(); String orchestrationName = request.getQueryParameters().get("orchestrationName"); - String instanceId = client.scheduleNewOrchestrationInstance(orchestrationName); + String instanceId = request.getQueryParameters().get("instanceId"); + instanceId = client.scheduleNewOrchestrationInstance(orchestrationName, "", instanceId); context.getLogger().info("Started orchestration with ID = '" + instanceId + "'."); return durableContext.createCheckStatusResponse(request, instanceId); } @@ -75,6 +76,7 @@ public HttpResponseMessage httpStartScheduled( } NewOrchestrationInstanceOptions startOptions = new NewOrchestrationInstanceOptions(); startOptions.setStartTime(scheduledStartTime); + startOptions.setInstanceId(request.getQueryParameters().get("instanceId")); String instanceId = client.scheduleNewOrchestrationInstance("HelloCities", startOptions); context.getLogger().info("Started orchestration with ID = '" + instanceId + "'."); return durableContext.createCheckStatusResponse(request, instanceId); diff --git a/test/e2e/Apps/BasicNode/host.json b/test/e2e/Apps/BasicNode/host.json index eb83303bc..0a6ad5997 100644 --- a/test/e2e/Apps/BasicNode/host.json +++ b/test/e2e/Apps/BasicNode/host.json @@ -9,7 +9,8 @@ }, "defaultVersion": "2.0", "versionMatchStrategy": "CurrentOrOlder", - "versionFailureStrategy": "Fail" + "versionFailureStrategy": "Fail", + "overridableExistingInstanceStates": "AnyState" } } } \ No newline at end of file diff --git a/test/e2e/Apps/BasicNode/src/functions/HelloCities.ts b/test/e2e/Apps/BasicNode/src/functions/HelloCities.ts index 5fc2b4aad..c9667ee3c 100644 --- a/test/e2e/Apps/BasicNode/src/functions/HelloCities.ts +++ b/test/e2e/Apps/BasicNode/src/functions/HelloCities.ts @@ -32,7 +32,8 @@ df.app.activity(activityName, { handler: HelloCitiesActivity }); const HelloCitiesHttpStartScheduled: HttpHandler = async (request: HttpRequest, context: InvocationContext): Promise => { const client = df.getClient(context); const body: unknown = await request.text(); - const instanceId: string = await client.startNew("HelloCities", { input: request.params.ScheduledStartTime }); + + const instanceId: string = await client.startNew("HelloCities", { input: request.params.ScheduledStartTime }); context.log(`Started orchestration with ID = '${instanceId}'.`); @@ -48,7 +49,8 @@ app.http('HelloCities_HttpStart_Scheduled', { const StartOrchestration: HttpHandler = async (request: HttpRequest, context: InvocationContext): Promise => { const client = df.getClient(context); - const instanceId: string = await client.startNew(request.params.orchestrationName); + + const instanceId = await client.startNew(request.params.orchestrationName, { instanceId: request.params.instanceId }); context.log(`Started orchestration with ID = '${instanceId}'.`); diff --git a/test/e2e/Apps/BasicPowerShell/StartOrchestration/run.ps1 b/test/e2e/Apps/BasicPowerShell/StartOrchestration/run.ps1 index b9313ff98..d39130fe3 100644 --- a/test/e2e/Apps/BasicPowerShell/StartOrchestration/run.ps1 +++ b/test/e2e/Apps/BasicPowerShell/StartOrchestration/run.ps1 @@ -8,7 +8,8 @@ using namespace System.Net param($Request, $TriggerMetadata) $orchestrationName = $Request.Query.orchestrationName -$InstanceId = Start-DurableOrchestration -FunctionName $orchestrationName +$InstanceId = $Request.Query.instanceId +$InstanceId = Start-DurableOrchestration -FunctionName $orchestrationName -InstanceId $InstanceId Write-Host "Started orchestration with ID = '$InstanceId'" $Response = New-DurableOrchestrationCheckStatusResponse -Request $Request -InstanceId $InstanceId diff --git a/test/e2e/Apps/BasicPowerShell/host.json b/test/e2e/Apps/BasicPowerShell/host.json index ecb594e30..ffe42bdd2 100644 --- a/test/e2e/Apps/BasicPowerShell/host.json +++ b/test/e2e/Apps/BasicPowerShell/host.json @@ -2,7 +2,8 @@ "version": "2.0", "extensions": { "durableTask": { - "hubName": "PowerShellE2ETaskHub" + "hubName": "PowerShellE2ETaskHub", + "overridableExistingInstanceStates": "AnyState" } }, "managedDependency": { diff --git a/test/e2e/Apps/BasicPython/hello_cities.py b/test/e2e/Apps/BasicPython/hello_cities.py index d01e065cd..4040b3c45 100644 --- a/test/e2e/Apps/BasicPython/hello_cities.py +++ b/test/e2e/Apps/BasicPython/hello_cities.py @@ -15,7 +15,7 @@ @bp.route(route="StartOrchestration") @bp.durable_client_input(client_name="client") async def http_start(req: func.HttpRequest, client): - instance_id = await client.start_new(req.params.get('orchestrationName')) + instance_id = await client.start_new(req.params.get('orchestrationName'), req.params.get('instanceId')) logging.info(f"Started orchestration with ID = '{instance_id}'.") return client.create_check_status_response(req, instance_id) @@ -24,7 +24,7 @@ async def http_start(req: func.HttpRequest, client): @bp.route(route="HelloCities_HttpStart_Scheduled") @bp.durable_client_input(client_name="client") async def http_start_scheduled(req: func.HttpRequest, client): - instance_id = await client.start_new('HelloCities', None, req.params.get('ScheduledStartTime')) + instance_id = await client.start_new('HelloCities', req.params.get('ScheduledStartTime')) logging.info(f"Started orchestration with ID = '{instance_id}'.") return client.create_check_status_response(req, instance_id) diff --git a/test/e2e/Apps/BasicPython/host.json b/test/e2e/Apps/BasicPython/host.json index 334ed1737..a7ea8cfc0 100644 --- a/test/e2e/Apps/BasicPython/host.json +++ b/test/e2e/Apps/BasicPython/host.json @@ -6,7 +6,8 @@ "tracing": { "DistributedTracingEnabled": true, "Version": "V2" - } + }, + "overridableExistingInstanceStates": "AnyState" } } } \ No newline at end of file diff --git a/test/e2e/Tests/DedupeStatusesTests.cs b/test/e2e/Tests/DedupeStatusesTests.cs deleted file mode 100644 index 5c62ece88..000000000 --- a/test/e2e/Tests/DedupeStatusesTests.cs +++ /dev/null @@ -1,12 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace Microsoft.Azure.Durable.Tests.DotnetIsolatedE2E -{ - internal class DedupeStatusesTests - { - } -} diff --git a/test/e2e/Tests/Tests/DedupeStatusesTests.cs b/test/e2e/Tests/Tests/DedupeStatusesTests.cs new file mode 100644 index 000000000..1e5ed9161 --- /dev/null +++ b/test/e2e/Tests/Tests/DedupeStatusesTests.cs @@ -0,0 +1,174 @@ +using System.Net; +using System.Text.Json; +using Xunit; +using Xunit.Abstractions; + +namespace Microsoft.Azure.Durable.Tests.DotnetIsolatedE2E; + +[Collection(Constants.FunctionAppCollectionSequentialName)] +public class DedupeStatusesTests +{ + private readonly FunctionAppFixture fixture; + private readonly ITestOutputHelper output; + + public DedupeStatusesTests(FunctionAppFixture fixture, ITestOutputHelper testOutputHelper) + { + this.fixture = fixture; + this.fixture.TestLogs.UseTestLogger(testOutputHelper); + this.output = testOutputHelper; + } + + [Fact] + public async Task CanStartOrchestrationWithSameIdForAllStatusesForEmptyDedupeStatuses() + { + // Completed + string completedInstanceId = Guid.NewGuid().ToString(); + using HttpResponseMessage startCompletedResponseFirstAttempt = await StartAndWaitForState("HelloCities", completedInstanceId, "Completed"); + using HttpResponseMessage startCompletedResponseSecondAttempt = await StartAndWaitForState("HelloCities", completedInstanceId, "Completed"); + + // Failed + string failedInstanceId = Guid.NewGuid().ToString(); + using HttpResponseMessage startFailedResponseFirstAttempt = await StartAndWaitForState("RethrowActivityException", failedInstanceId, "Failed"); + // Invoking this same orchestration with the same instance ID will cause it to complete successfully on the second attempt, hence we look for a "Completed" status instead + using HttpResponseMessage startFailedResponseSecondAttempt = await StartAndWaitForState("RethrowActivityException", failedInstanceId, "Completed"); + + // Terminated + if (this.fixture.functionLanguageLocalizer.GetLanguageType() != LanguageType.Java + || this.fixture.GetDurabilityProvider() != FunctionAppFixture.ConfiguredDurabilityProviderType.MSSQL) // Bug: https://github.com/microsoft/durabletask-java/issues/237 + { + string terminatedInstanceId = Guid.NewGuid().ToString(); + using HttpResponseMessage startTerminatedResponseFirstAttempt = await StartAndWaitForState("LongRunningOrchestrator", terminatedInstanceId, "Running"); + await TerminateAndWaitForState(terminatedInstanceId, startTerminatedResponseFirstAttempt); + using HttpResponseMessage startTerminatedResponseSecondAttempt = await StartAndWaitForState("LongRunningOrchestrator", terminatedInstanceId, "Running"); + } + + // Pending + // Scheduled start times are currently only implemented in Java and .NET isolated, which is the only true way to get an orchestration in a "Pending" state + if (this.fixture.functionLanguageLocalizer.GetLanguageType() == LanguageType.DotnetIsolated + || this.fixture.functionLanguageLocalizer.GetLanguageType() == LanguageType.Java) + { + string pendingInstanceId = Guid.NewGuid().ToString(); + DateTime scheduledStartTime = DateTime.UtcNow.AddMinutes(2); + using HttpResponseMessage startPendingResponseFirstAttempt = await StartAndWaitForState("HelloCities", pendingInstanceId, "Pending", scheduledStartTime: scheduledStartTime); + using HttpResponseMessage startPendingResponseSecondAttempt = await StartAndWaitForState("HelloCities", pendingInstanceId, "Completed"); + } + + // Running + string runningInstanceId = Guid.NewGuid().ToString(); + using HttpResponseMessage startRunningResponseFirstAttempt = await StartAndWaitForState("LongRunningOrchestrator", runningInstanceId, "Running"); + using HttpResponseMessage startRunningResponseSecondAttempt = await StartAndWaitForState("LongRunningOrchestrator", runningInstanceId, "Running"); + + // Suspended + string suspendedInstanceId = Guid.NewGuid().ToString(); + using HttpResponseMessage startSuspendedResponseFirstAttempt = await StartAndWaitForState("LongRunningOrchestrator", suspendedInstanceId, "Running"); + await SuspendAndWaitForState(suspendedInstanceId, startSuspendedResponseFirstAttempt); + using HttpResponseMessage startSuspendedResponseSecondAttempt = await StartAndWaitForState("LongRunningOrchestrator", suspendedInstanceId, "Running"); + } + + [Fact] + [Trait("PowerShell", "Skip")] // Dedupe statuses not implemented in PowerShell + [Trait("Python", "Skip")] // Dedupe statuses not implemented in Python + [Trait("Node", "Skip")] // Dedupe statuses not implemented in Node + [Trait("Java", "Skip")] // Dedupe statuses not implemented in Java + public async Task StartOrchestrationWithSameIdFailsForDedupeStatuses() + { + List dedupeStatuses = ["Running", "Failed"]; + + // Completed, should succeed + string completedInstanceId = Guid.NewGuid().ToString(); + using HttpResponseMessage startCompletedResponseFirstAttempt = await StartAndWaitForStateWithDedupeStatuses("HelloCities", completedInstanceId, "Completed", dedupeStatuses); + using HttpResponseMessage startCompletedResponseSecondAttempt = await StartAndWaitForStateWithDedupeStatuses("HelloCities", completedInstanceId, "Completed", dedupeStatuses); + + // Terminated + string terminatedInstanceId = Guid.NewGuid().ToString(); + using HttpResponseMessage startTerminatedResponseFirstAttempt = await StartAndWaitForStateWithDedupeStatuses("LongRunningOrchestrator", terminatedInstanceId, "Running", dedupeStatuses); + await TerminateAndWaitForState(terminatedInstanceId, startTerminatedResponseFirstAttempt); + using HttpResponseMessage startTerminatedResponseSecondAttempt = await StartAndWaitForStateWithDedupeStatuses("LongRunningOrchestrator", terminatedInstanceId, "Running", dedupeStatuses); + + // Pending + string pendingInstanceId = Guid.NewGuid().ToString(); + DateTime scheduledStartTime = DateTime.UtcNow.AddMinutes(2); + using HttpResponseMessage startPendingResponseFirstAttempt = await StartAndWaitForStateWithDedupeStatuses("HelloCities", pendingInstanceId, "Pending", dedupeStatuses, scheduledStartTime: scheduledStartTime); + using HttpResponseMessage startPendingResponseSecondAttempt = await StartAndWaitForStateWithDedupeStatuses("HelloCities", pendingInstanceId, "Completed", dedupeStatuses); + + // Suspended + string suspendedInstanceId = Guid.NewGuid().ToString(); + using HttpResponseMessage startSuspendedResponseFirstAttempt = await StartAndWaitForStateWithDedupeStatuses("LongRunningOrchestrator", suspendedInstanceId, "Running", dedupeStatuses); + await SuspendAndWaitForState(suspendedInstanceId, startSuspendedResponseFirstAttempt); + using HttpResponseMessage startSuspendedResponseSecondAttempt = await StartAndWaitForStateWithDedupeStatuses("LongRunningOrchestrator", suspendedInstanceId, "Running", dedupeStatuses); + + // Failed, should fail + string failedInstanceId = Guid.NewGuid().ToString(); + using HttpResponseMessage startFailedResponseFirstAttempt = await StartAndWaitForStateWithDedupeStatuses("RethrowActivityException", failedInstanceId, "Failed", dedupeStatuses); + using HttpResponseMessage startFailedResponseSecondAttempt = await StartAndWaitForStateWithDedupeStatuses("RethrowActivityException", failedInstanceId, "Failed", dedupeStatuses, expectedCode: HttpStatusCode.BadRequest); + + // Running, should fail + string runningInstanceId = Guid.NewGuid().ToString(); + using HttpResponseMessage startRunningResponseFirstAttempt = await StartAndWaitForStateWithDedupeStatuses("LongRunningOrchestrator", runningInstanceId, "Running", dedupeStatuses); + using HttpResponseMessage startRunningResponseSecondAttempt = await StartAndWaitForStateWithDedupeStatuses("LongRunningOrchestrator", runningInstanceId, "Running", dedupeStatuses, expectedCode: HttpStatusCode.BadRequest); + } + + private static async Task StartAndWaitForState( + string orchestrationName, + string instanceId, + string expectedState, + DateTime? scheduledStartTime = null) + { + string functionName = "StartOrchestration"; + string queryString = $"?orchestrationName={orchestrationName}&instanceId={instanceId}"; + + if (scheduledStartTime is not null) + { + queryString += $"&scheduledStartTime={scheduledStartTime:o}"; + functionName = "HelloCities_HttpStart_Scheduled"; + } + + HttpResponseMessage response = await HttpHelpers.InvokeHttpTrigger(functionName, queryString); + Assert.Equal(HttpStatusCode.Accepted, response.StatusCode); + string statusQueryGetUri = await DurableHelpers.ParseStatusQueryGetUriAsync(response); + await DurableHelpers.WaitForOrchestrationStateAsync(statusQueryGetUri, expectedState, 60); + return response; + } + + private static async Task StartAndWaitForStateWithDedupeStatuses( + string orchestrationName, + string instanceId, + string expectedState, + List dedupeStatuses, + DateTime? scheduledStartTime = null, + HttpStatusCode expectedCode = HttpStatusCode.Accepted) + { + string queryString = $"?orchestrationName={orchestrationName}&instanceId={instanceId}&dedupeStatuses={JsonSerializer.Serialize(dedupeStatuses)}"; + + if (scheduledStartTime is not null) + { + queryString += $"&scheduledStartTime={scheduledStartTime:o}"; + } + + HttpResponseMessage response = await HttpHelpers.InvokeHttpTrigger("StartOrchestration_DedupeStatuses", queryString); + Assert.Equal(expectedCode, response.StatusCode); + if (expectedCode != HttpStatusCode.Accepted) + { + return response; + } + string statusQueryGetUri = await DurableHelpers.ParseStatusQueryGetUriAsync(response); + await DurableHelpers.WaitForOrchestrationStateAsync(statusQueryGetUri, expectedState, 60); + return response; + } + + private static async Task TerminateAndWaitForState(string instanceId, HttpResponseMessage startOrchestrationResponse) + { + using HttpResponseMessage terminateResponse = await HttpHelpers.InvokeHttpTrigger("TerminateInstance", $"?instanceId={instanceId}"); + Assert.Equal(HttpStatusCode.OK, terminateResponse.StatusCode); + string statusQueryGetUri = await DurableHelpers.ParseStatusQueryGetUriAsync(startOrchestrationResponse); + await DurableHelpers.WaitForOrchestrationStateAsync(statusQueryGetUri, "Terminated", 60); + } + + private static async Task SuspendAndWaitForState(string instanceId, HttpResponseMessage startOrchestrationResponse) + { + using HttpResponseMessage suspendResponse = await HttpHelpers.InvokeHttpTrigger("SuspendInstance", $"?instanceId={instanceId}"); + Assert.Equal(HttpStatusCode.OK, suspendResponse.StatusCode); + string statusQueryGetUri = await DurableHelpers.ParseStatusQueryGetUriAsync(startOrchestrationResponse); + await DurableHelpers.WaitForOrchestrationStateAsync(statusQueryGetUri, "Suspended", 60); + } +} diff --git a/test/e2e/Tests/Tests/DistributedTracingEntitiesTests.cs b/test/e2e/Tests/Tests/DistributedTracingEntitiesTests.cs index f538a14df..fbc6f3124 100644 --- a/test/e2e/Tests/Tests/DistributedTracingEntitiesTests.cs +++ b/test/e2e/Tests/Tests/DistributedTracingEntitiesTests.cs @@ -57,8 +57,7 @@ public async Task DistributedTracingEntitiesTest() var orchestrationDetails = await DurableHelpers.GetRunningOrchestrationDetailsAsync(statusQueryGetUri); // Sanitize the JSON string to remove unwanted characters so we can easily parse it into a list - var output = orchestrationDetails.Output.Replace("\r", "").Replace("\n", "").Replace("\"", "").Replace("[", "").Replace("]", "").Replace(" ", ""); - var ids = new List(output.Split(",")); + var ids = JsonSerializer.Deserialize>(orchestrationDetails.Output); // The execution is as follows: // Orchestration A signals entity A which signals entity B. Then orchestration A calls entities A and B. Finally orchestration A signals entity C. @@ -67,6 +66,7 @@ public async Task DistributedTracingEntitiesTest() // Orchestration A and B return this list of Activities as part of their output. In order to access the output of orchestration B, we need to return its // instance ID as part of the output of orchestration A. It will be the last item in the list returned by A, so we will remove it from the list and use it // to get the output of orchestration B (which will have the final two Activities, that for orchestration B and its call to entity A). + Assert.NotNull(ids); Assert.Equal(7, ids.Count); var orchestrationId = ids[ids.Count - 1]; ids.RemoveAt(ids.Count - 1); diff --git a/test/e2e/Tests/Tests/PurgeInstancesTests.cs b/test/e2e/Tests/Tests/PurgeInstancesTests.cs index 1b861a12e..c9e55d5fd 100644 --- a/test/e2e/Tests/Tests/PurgeInstancesTests.cs +++ b/test/e2e/Tests/Tests/PurgeInstancesTests.cs @@ -161,7 +161,7 @@ void AssertFailedPurgeResponseStatusCode(HttpResponseMessage purgeHttpResponse) // Terminated orchestration, should succeed if (this.fixture.functionLanguageLocalizer.GetLanguageType() != LanguageType.Java - && this.fixture.GetDurabilityProvider() != FunctionAppFixture.ConfiguredDurabilityProviderType.MSSQL) // Bug: https://github.com/microsoft/durabletask-java/issues/237 + || this.fixture.GetDurabilityProvider() != FunctionAppFixture.ConfiguredDurabilityProviderType.MSSQL) // Bug: https://github.com/microsoft/durabletask-java/issues/237 { using HttpResponseMessage startTerminatedOrchestrationResponse = await HttpHelpers.InvokeHttpTrigger( "StartOrchestration", @@ -207,16 +207,21 @@ void AssertFailedPurgeResponseStatusCode(HttpResponseMessage purgeHttpResponse) AssertFailedPurgeResponseStatusCode(purgeRunning); // Pending orchestration, should fail - DateTime scheduledStartTime = DateTime.UtcNow + TimeSpan.FromMinutes(1); - using HttpResponseMessage startPendingOrchestrationResponse = await HttpHelpers.InvokeHttpTrigger( - "HelloCities_HttpStart_Scheduled", - $"?ScheduledStartTime={scheduledStartTime:o}"); - Assert.Equal(HttpStatusCode.Accepted, startPendingOrchestrationResponse.StatusCode); - string pendingInstanceId = await DurableHelpers.ParseInstanceIdAsync(startPendingOrchestrationResponse); - string pendingStatusQueryGetUri = await DurableHelpers.ParseStatusQueryGetUriAsync(startPendingOrchestrationResponse); - await DurableHelpers.WaitForOrchestrationStateAsync(pendingStatusQueryGetUri, "Pending", 30); - using HttpResponseMessage purgePending = await HttpHelpers.InvokeHttpTrigger("PurgeOrchestrationHistory", $"?instanceId={pendingInstanceId}"); - AssertFailedPurgeResponseStatusCode(purgePending); + // Scheduled start times are currently only implemented in Java and .NET isolated, which is the only true way to get an orchestration in a "Pending" state + if (this.fixture.functionLanguageLocalizer.GetLanguageType() == LanguageType.DotnetIsolated + || this.fixture.functionLanguageLocalizer.GetLanguageType() == LanguageType.Java) + { + DateTime scheduledStartTime = DateTime.UtcNow + TimeSpan.FromMinutes(1); + using HttpResponseMessage startPendingOrchestrationResponse = await HttpHelpers.InvokeHttpTrigger( + "HelloCities_HttpStart_Scheduled", + $"?ScheduledStartTime={scheduledStartTime:o}"); + Assert.Equal(HttpStatusCode.Accepted, startPendingOrchestrationResponse.StatusCode); + string pendingInstanceId = await DurableHelpers.ParseInstanceIdAsync(startPendingOrchestrationResponse); + string pendingStatusQueryGetUri = await DurableHelpers.ParseStatusQueryGetUriAsync(startPendingOrchestrationResponse); + await DurableHelpers.WaitForOrchestrationStateAsync(pendingStatusQueryGetUri, "Pending", 30); + using HttpResponseMessage purgePending = await HttpHelpers.InvokeHttpTrigger("PurgeOrchestrationHistory", $"?instanceId={pendingInstanceId}"); + AssertFailedPurgeResponseStatusCode(purgePending); + } // Suspended orchestration, should fail using HttpResponseMessage startSuspendedOrchestrationResponse = await HttpHelpers.InvokeHttpTrigger( From 56c9551439580a0f31a245b6656779f96c031043 Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Fri, 16 Jan 2026 22:41:01 -0800 Subject: [PATCH 03/43] removing unnecessary changes --- .../OverridableStates.cs | 114 +++++++++--------- .../TaskHubGrpcServer.cs | 83 ------------- test/Common/TestHelpers.cs | 24 ++-- test/e2e/Apps/BasicPython/hello_cities.py | 2 +- 4 files changed, 70 insertions(+), 153 deletions(-) diff --git a/src/WebJobs.Extensions.DurableTask/OverridableStates.cs b/src/WebJobs.Extensions.DurableTask/OverridableStates.cs index acef367ae..dde583d58 100644 --- a/src/WebJobs.Extensions.DurableTask/OverridableStates.cs +++ b/src/WebJobs.Extensions.DurableTask/OverridableStates.cs @@ -1,57 +1,57 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the MIT License. See LICENSE in the project root for license information. - -using System; -using DurableTask.Core; - -namespace Microsoft.Azure.WebJobs.Extensions.DurableTask -{ - /// - /// Represents options for different states that an existing orchestrator can be in to be able to be overwritten by - /// an attempt to start a new instance with the same instance Id. - /// - public enum OverridableStates - { - /// - /// Option to start a new orchestrator instance with an existing instnace Id when the existing - /// instance is in any state. - /// - AnyState, - - /// - /// Option to only start a new orchestrator instance with an existing instance Id when the existing - /// instance is in a terminated, failed, or completed state. - /// - NonRunningStates, - } - - /// - /// Extension methods for . - /// -#pragma warning disable SA1649 // File name should match first type name Justification: pairing extension methods with enum. - internal static class OverridableStatesExtensions -#pragma warning restore SA1649 // File name should match first type name - { - private static readonly OrchestrationStatus[] NonRunning = new OrchestrationStatus[] - { - OrchestrationStatus.Running, - OrchestrationStatus.ContinuedAsNew, - OrchestrationStatus.Pending, - OrchestrationStatus.Suspended, - }; - - /// - /// Gets the dedupe for a given . - /// - /// The overridable states. - /// An array of statuses to dedupe. - public static OrchestrationStatus[] ToDedupeStatuses(this OverridableStates states) - { - return states switch - { - OverridableStates.NonRunningStates => NonRunning, - _ => Array.Empty(), - }; - } - } -} +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See LICENSE in the project root for license information. + +using System; +using DurableTask.Core; + +namespace Microsoft.Azure.WebJobs.Extensions.DurableTask +{ + /// + /// Represents options for different states that an existing orchestrator can be in to be able to be overwritten by + /// an attempt to start a new instance with the same instance Id. + /// + public enum OverridableStates + { + /// + /// Option to start a new orchestrator instance with an existing instnace Id when the existing + /// instance is in any state. + /// + AnyState, + + /// + /// Option to only start a new orchestrator instance with an existing instance Id when the existing + /// instance is in a terminated, failed, or completed state. + /// + NonRunningStates, + } + + /// + /// Extension methods for . + /// +#pragma warning disable SA1649 // File name should match first type name Justification: pairing extension methods with enum. + internal static class OverridableStatesExtensions +#pragma warning restore SA1649 // File name should match first type name + { + private static readonly OrchestrationStatus[] NonRunning = new OrchestrationStatus[] + { + OrchestrationStatus.Running, + OrchestrationStatus.ContinuedAsNew, + OrchestrationStatus.Pending, + OrchestrationStatus.Suspended, + }; + + /// + /// Gets the dedupe for a given . + /// + /// The overridable states. + /// An array of statuses to dedupe. + public static OrchestrationStatus[] ToDedupeStatuses(this OverridableStates states) + { + return states switch + { + OverridableStates.NonRunningStates => NonRunning, + _ => Array.Empty(), + }; + } + } +} diff --git a/src/WebJobs.Extensions.DurableTask/TaskHubGrpcServer.cs b/src/WebJobs.Extensions.DurableTask/TaskHubGrpcServer.cs index 14b8f1c2f..64b7e14c3 100644 --- a/src/WebJobs.Extensions.DurableTask/TaskHubGrpcServer.cs +++ b/src/WebJobs.Extensions.DurableTask/TaskHubGrpcServer.cs @@ -689,88 +689,5 @@ private static bool IsOrchestrationCompleted(OrchestrationStatus status) status == OrchestrationStatus.Terminated || status == OrchestrationStatus.Failed; } - - private async Task TerminateExistingNonTerminalInstance(P.CreateInstanceRequest request, ServerCallContext context, IEnumerable dedupeStatuses) - { - var runningStatuses = new List() - { - P.OrchestrationStatus.Running, - P.OrchestrationStatus.Pending, - P.OrchestrationStatus.Suspended, - }; - - // Only check if an existing instance with this instance ID is running if: - // 1. The user explicitly specified in their options settings that any existing instance state is overridable - // (as opposed to only terminal states), and - // 2. The reusable statuses passed to the creation request contain at least one non-terminal status. - if (this.extension.Options.OverridableExistingInstanceStates == OverridableStates.AnyState - && request.OrchestrationIdReusePolicy.ReplaceableStatus.Any(status => runningStatuses.Contains(status))) - { - OrchestrationState orchestrationState = await this.GetDurabilityProvider(context) - .GetOrchestrationStateAsync(request.InstanceId, executionId: null); - - if (orchestrationState?.OrchestrationInstance != null) - { - // If an existing instance is found, check if its status is in the list of reusable statuses. - if (!request.OrchestrationIdReusePolicy.ReplaceableStatus.Contains((P.OrchestrationStatus)orchestrationState.OrchestrationStatus)) - { - throw new OrchestrationAlreadyExistsException(); - } - - // If the existing instance is in a non-terminal state, terminate it before creating a new instance. - if (runningStatuses.Contains((P.OrchestrationStatus)orchestrationState.OrchestrationStatus)) - { - OrchestrationStatus originalStatus = orchestrationState.OrchestrationStatus; - - // Check for cancellation before attempting to terminate - if (context.CancellationToken.IsCancellationRequested) - { - throw new RpcException(new Status(StatusCode.Cancelled, $"Create instance request cancelled for instance ID {request.InstanceId}")); - } - - await this.GetClient(context).TerminateAsync(request.InstanceId, $"A new instance creation request has been issued for instance " + - $"{request.InstanceId}, which currently has status {orchestrationState.OrchestrationStatus}. Since the " + - $"{nameof(DurableTaskOptions.OverridableExistingInstanceStates)} is set to {nameof(OverridableStates.AnyState)}, and the dedupe " + - $"statuses of the creation request, {dedupeStatuses}, do not contain the orchestration's status, the orchestration has been " + - $"terminated and a new instance with the same instance ID will be created."); - - var timeoutCts = new CancellationTokenSource(TimeSpan.FromSeconds(90)); - using var linkedCts = - CancellationTokenSource.CreateLinkedTokenSource( - context.CancellationToken, - timeoutCts.Token); - - while (!linkedCts.IsCancellationRequested - && orchestrationState != null - && orchestrationState.OrchestrationStatus != OrchestrationStatus.Terminated) - { - await Task.Delay(TimeSpan.FromSeconds(1), context.CancellationToken); - orchestrationState = await this.GetDurabilityProvider(context) - .GetOrchestrationStateAsync(request.InstanceId, executionId: null); - } - - if (linkedCts.IsCancellationRequested) - { - throw new RpcException(new Status( - StatusCode.Cancelled, - context.CancellationToken.IsCancellationRequested - ? $"Create instance request cancelled for instance ID {request.InstanceId}" - : $"Create instance request exceeded timeout of 100 seconds for instance ID {request.InstanceId} " + - $"while waiting for the termination of the existing instance with this instance ID.")); - } - - this.extension.TraceHelper.ExtensionInformationalEvent( - this.extension.Options.HubName, - functionName: request.Name, - instanceId: request.InstanceId, - message: $"Successfully terminated existing instance with instance ID {request.InstanceId} and status {originalStatus} " + - $"in create instance request. Will proceed to create a new instance with this ID.", - writeToUserLogs: true); - - // What should we do if the dedupe statuses contain the Terminated status? Then the creation request after this point will fail. Should we remove this status? - } - } - } - } } } diff --git a/test/Common/TestHelpers.cs b/test/Common/TestHelpers.cs index e407a4db4..3a85dc875 100644 --- a/test/Common/TestHelpers.cs +++ b/test/Common/TestHelpers.cs @@ -412,15 +412,15 @@ public static Task DeleteTaskHubResources(string testName, bool enableExtendedSe var service = new AzureStorageOrchestrationService(settings); return service.DeleteAsync(); - } - - public static void AssertLogMessageSequence( - ITestOutputHelper testOutput, - TestLoggerProvider loggerProvider, - string testName, - string instanceId, - bool filterOutReplayLogs, - string[] orchestratorFunctionNames, + } + + public static void AssertLogMessageSequence( + ITestOutputHelper testOutput, + TestLoggerProvider loggerProvider, + string testName, + string instanceId, + bool filterOutReplayLogs, + string[] orchestratorFunctionNames, string activityFunctionName = null) { List messageIds; @@ -488,9 +488,9 @@ private static List GetLogMessages( { // It is assumed that the 5th log message is a sub-orchestration instanceIds.Add(GetInstanceId(logMessages[4].FormattedMessage)); - } - - Assert.True( + } + + Assert.True( logMessages.TrueForAll(m => m.Category.Equals(LogCategory, StringComparison.InvariantCultureIgnoreCase))); return logMessages; diff --git a/test/e2e/Apps/BasicPython/hello_cities.py b/test/e2e/Apps/BasicPython/hello_cities.py index 4040b3c45..395afd186 100644 --- a/test/e2e/Apps/BasicPython/hello_cities.py +++ b/test/e2e/Apps/BasicPython/hello_cities.py @@ -24,7 +24,7 @@ async def http_start(req: func.HttpRequest, client): @bp.route(route="HelloCities_HttpStart_Scheduled") @bp.durable_client_input(client_name="client") async def http_start_scheduled(req: func.HttpRequest, client): - instance_id = await client.start_new('HelloCities', req.params.get('ScheduledStartTime')) + instance_id = await client.start_new('HelloCities', None, req.params.get('ScheduledStartTime')) logging.info(f"Started orchestration with ID = '{instance_id}'.") return client.create_check_status_response(req, instance_id) From 9145d446c44e252873fdae23ee6872ef469bae24 Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Fri, 16 Jan 2026 22:44:04 -0800 Subject: [PATCH 04/43] made hellocities more succinct --- test/e2e/Apps/BasicDotNetIsolated/HelloCities.cs | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/test/e2e/Apps/BasicDotNetIsolated/HelloCities.cs b/test/e2e/Apps/BasicDotNetIsolated/HelloCities.cs index ced2caa5f..c6837bb4d 100644 --- a/test/e2e/Apps/BasicDotNetIsolated/HelloCities.cs +++ b/test/e2e/Apps/BasicDotNetIsolated/HelloCities.cs @@ -49,14 +49,7 @@ public static async Task StartOrchestration( ILogger logger = executionContext.GetLogger(nameof(StartOrchestration)); // Function input comes from the request content. - if (instanceId is not null) - { - await client.ScheduleNewOrchestrationInstanceAsync(orchestrationName, new StartOrchestrationOptions(InstanceId: instanceId)); - } - else - { - instanceId = await client.ScheduleNewOrchestrationInstanceAsync(orchestrationName); - } + await client.ScheduleNewOrchestrationInstanceAsync(orchestrationName, new StartOrchestrationOptions(InstanceId: instanceId)); logger.LogInformation("Started orchestration with ID = '{instanceId}'.", instanceId); @@ -75,13 +68,9 @@ public static async Task HttpStartScheduled( { ILogger logger = executionContext.GetLogger("HelloCities_HttpStart"); - var startOptions = new StartOrchestrationOptions(StartAt: scheduledStartTime); + var startOptions = new StartOrchestrationOptions(StartAt: scheduledStartTime, InstanceId: instanceId); // Function input comes from the request content. - if (instanceId is not null) - { - startOptions = startOptions with { InstanceId = instanceId }; - } instanceId = await client.ScheduleNewOrchestrationInstanceAsync( nameof(HelloCities), startOptions); From c1a7805dae4b9575cba244691c29ee5f14e82677 Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Fri, 16 Jan 2026 23:28:06 -0800 Subject: [PATCH 05/43] missed instance ID assignment --- test/e2e/Apps/BasicDotNetIsolated/HelloCities.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/Apps/BasicDotNetIsolated/HelloCities.cs b/test/e2e/Apps/BasicDotNetIsolated/HelloCities.cs index c6837bb4d..67b30e399 100644 --- a/test/e2e/Apps/BasicDotNetIsolated/HelloCities.cs +++ b/test/e2e/Apps/BasicDotNetIsolated/HelloCities.cs @@ -49,7 +49,7 @@ public static async Task StartOrchestration( ILogger logger = executionContext.GetLogger(nameof(StartOrchestration)); // Function input comes from the request content. - await client.ScheduleNewOrchestrationInstanceAsync(orchestrationName, new StartOrchestrationOptions(InstanceId: instanceId)); + instanceId = await client.ScheduleNewOrchestrationInstanceAsync(orchestrationName, new StartOrchestrationOptions(InstanceId: instanceId)); logger.LogInformation("Started orchestration with ID = '{instanceId}'.", instanceId); From 7bfde01304431941bfd083c3e947bcb78307ef4b Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Tue, 20 Jan 2026 18:03:36 -0800 Subject: [PATCH 06/43] PR comment updates --- .../DurabilityProvider.cs | 2 +- .../TaskHubGrpcServer.cs | 186 +++++++++--------- test/Common/DurableTaskEndToEndTests.cs | 13 +- .../Apps/BasicDotNetIsolated/HelloCities.cs | 2 +- test/e2e/Tests/Tests/DedupeStatusesTests.cs | 2 +- 5 files changed, 101 insertions(+), 104 deletions(-) diff --git a/src/WebJobs.Extensions.DurableTask/DurabilityProvider.cs b/src/WebJobs.Extensions.DurableTask/DurabilityProvider.cs index 2fe264db2..2f4645059 100644 --- a/src/WebJobs.Extensions.DurableTask/DurabilityProvider.cs +++ b/src/WebJobs.Extensions.DurableTask/DurabilityProvider.cs @@ -435,7 +435,7 @@ await this.TerminateTaskOrchestrationWithReusableRunningStatusAndWaitAsync( /// will be thrown. /// If the array contains all of the running statuses (, , /// and ), then only terminal statuses can be reused. - /// If at least one of these statuses is not included in the array, then if an instance with that status is found, it will first be terminated via + /// If at least one of these statuses is not included in the array, then if an instance with that status is found, it will first be terminated /// before a new orchestration is created. /// The cancellation token. /// A task that completes when the creation message for the task orchestration instance is enqueued. diff --git a/src/WebJobs.Extensions.DurableTask/TaskHubGrpcServer.cs b/src/WebJobs.Extensions.DurableTask/TaskHubGrpcServer.cs index 64b7e14c3..8a23db56c 100644 --- a/src/WebJobs.Extensions.DurableTask/TaskHubGrpcServer.cs +++ b/src/WebJobs.Extensions.DurableTask/TaskHubGrpcServer.cs @@ -53,7 +53,7 @@ public override Task Hello(Empty request, ServerCallContext context) public async override Task StartInstance(P.CreateInstanceRequest request, ServerCallContext context) { try - { + { var allStatuses = new List() { OrchestrationStatus.Running, @@ -70,10 +70,10 @@ public override Task Hello(Empty request, ServerCallContext context) List reusableStatuses = request.OrchestrationIdReusePolicy is null ? allStatuses : request.OrchestrationIdReusePolicy.ReplaceableStatus.Select(status => (OrchestrationStatus)status).ToList(); - - OrchestrationStatus[] dedupeStatuses = allStatuses - .Except(reusableStatuses) - .Union(this.extension.Options.OverridableExistingInstanceStates.ToDedupeStatuses()) + + OrchestrationStatus[] dedupeStatuses = allStatuses + .Except(reusableStatuses) + .Union(this.extension.Options.OverridableExistingInstanceStates.ToDedupeStatuses()) .ToArray(); // Create the orchestration instance @@ -99,7 +99,7 @@ public override Task Hello(Empty request, ServerCallContext context) // Create a new activity with the parent context ActivityContext.TryParse(traceParent, traceState, out ActivityContext parentActivityContext); - using Activity? scheduleOrchestrationActivity = TraceHelper.StartActivityForNewOrchestration(executionStartedEvent, parentActivityContext, request.RequestTime?.ToDateTimeOffset()); + using Activity? scheduleOrchestrationActivity = TraceHelper.StartActivityForNewOrchestration(executionStartedEvent, parentActivityContext, request.RequestTime?.ToDateTimeOffset()); // Schedule the orchestration await this.GetDurabilityProvider(context).CreateTaskOrchestrationAsync( @@ -108,7 +108,7 @@ await this.GetDurabilityProvider(context).CreateTaskOrchestrationAsync( Event = executionStartedEvent, OrchestrationInstance = instance, }, - dedupeStatuses, + dedupeStatuses, context.CancellationToken); return new P.CreateInstanceResponse @@ -123,8 +123,8 @@ await this.GetDurabilityProvider(context).CreateTaskOrchestrationAsync( catch (InvalidOperationException ex) when (ex.Message.EndsWith("already exists.")) // for older versions of DTF.AS and DTFx.Netherite { throw new RpcException(new Status(StatusCode.AlreadyExists, $"An Orchestration instance with the ID {request.InstanceId} already exists.")); - } - catch (OperationCanceledException) + } + catch (OperationCanceledException) { throw new RpcException(new Status( StatusCode.Cancelled, @@ -332,11 +332,11 @@ await this.GetDurabilityProvider(context).CreateTaskOrchestrationAsync( public async override Task RewindInstance(P.RewindInstanceRequest request, ServerCallContext context) { - try - { + try + { #pragma warning disable CS0618 // Type or member is obsolete await this.GetClient(context).RewindAsync(request.InstanceId, request.Reason); -#pragma warning restore CS0618 // Type or member is obsolete +#pragma warning restore CS0618 // Type or member is obsolete } catch (ArgumentException ex) { @@ -543,87 +543,87 @@ private static P.GetInstanceResponse CreateGetInstanceResponse(OrchestrationStat }; } - public async override Task StreamInstanceHistory( - P.StreamInstanceHistoryRequest request, - IServerStreamWriter responseStream, - ServerCallContext context) - { - if (await this.GetClient(context).GetStatusAsync(request.InstanceId, showInput: false) is null) - { - throw new RpcException(new Status(StatusCode.NotFound, $"Orchestration instance with ID {request.InstanceId} was not found.")); - } - - async Task<(P.HistoryChunk, int)> AddToHistoryChunkAndStream(HistoryEvent historyEvent, P.HistoryChunk historyChunk, int currentChunkSizeInBytes) - { - P.HistoryEvent result = ProtobufUtils.ToHistoryEventProto(historyEvent); - - int currentEventSize = result.CalculateSize(); - if (currentChunkSizeInBytes + currentEventSize > MaxHistoryChunkSizeInBytes) - { - // If we exceeded the chunk size threshold, send what we have so far. - await responseStream.WriteAsync(historyChunk); - historyChunk = new (); - currentChunkSizeInBytes = 0; - } - - historyChunk.Events.Add(result); - currentChunkSizeInBytes += currentEventSize; - return (historyChunk, currentChunkSizeInBytes); - } - - try - { - int currentChunkSizeInBytes = 0; - P.HistoryChunk historyChunk = new (); - - // First, try to use the streaming API if it's implemented. - try - { - IAsyncEnumerable historyEvents = await this.GetDurabilityProvider(context).StreamOrchestrationHistoryAsync( - request.InstanceId, - context.CancellationToken); - - await foreach (HistoryEvent historyEvent in historyEvents) - { - (historyChunk, currentChunkSizeInBytes) = await AddToHistoryChunkAndStream(historyEvent, historyChunk, currentChunkSizeInBytes); - } - } - - // Otherwise default to the older non-streaming implementation. - catch (NotImplementedException) - { - string jsonHistory = await this.GetDurabilityProvider(context).GetOrchestrationHistoryAsync( - request.InstanceId, - executionId: null); - - List? historyEvents = JsonConvert.DeserializeObject>( - jsonHistory, - new JsonSerializerSettings() - { - Converters = { new HistoryEventJsonConverter() }, - }) - ?? throw new Exception($"Failed to deserialize orchestration history."); - - foreach (HistoryEvent historyEvent in historyEvents) - { - (historyChunk, currentChunkSizeInBytes) = await AddToHistoryChunkAndStream(historyEvent, historyChunk, currentChunkSizeInBytes); - } - } - - // Send the last chunk, which may be smaller than the maximum chunk size. - if (historyChunk.Events.Count > 0) - { - await responseStream.WriteAsync(historyChunk); - } - } - catch (OperationCanceledException) - { - throw new RpcException(new Status(StatusCode.Cancelled, $"Orchestration history streaming cancelled for instance {request.InstanceId}")); - } - catch (Exception ex) - { - throw new RpcException(new Status(StatusCode.Internal, $"Failed to stream orchestration history for instance {request.InstanceId}: {ex.Message}")); - } + public async override Task StreamInstanceHistory( + P.StreamInstanceHistoryRequest request, + IServerStreamWriter responseStream, + ServerCallContext context) + { + if (await this.GetClient(context).GetStatusAsync(request.InstanceId, showInput: false) is null) + { + throw new RpcException(new Status(StatusCode.NotFound, $"Orchestration instance with ID {request.InstanceId} was not found.")); + } + + async Task<(P.HistoryChunk, int)> AddToHistoryChunkAndStream(HistoryEvent historyEvent, P.HistoryChunk historyChunk, int currentChunkSizeInBytes) + { + P.HistoryEvent result = ProtobufUtils.ToHistoryEventProto(historyEvent); + + int currentEventSize = result.CalculateSize(); + if (currentChunkSizeInBytes + currentEventSize > MaxHistoryChunkSizeInBytes) + { + // If we exceeded the chunk size threshold, send what we have so far. + await responseStream.WriteAsync(historyChunk); + historyChunk = new (); + currentChunkSizeInBytes = 0; + } + + historyChunk.Events.Add(result); + currentChunkSizeInBytes += currentEventSize; + return (historyChunk, currentChunkSizeInBytes); + } + + try + { + int currentChunkSizeInBytes = 0; + P.HistoryChunk historyChunk = new (); + + // First, try to use the streaming API if it's implemented. + try + { + IAsyncEnumerable historyEvents = await this.GetDurabilityProvider(context).StreamOrchestrationHistoryAsync( + request.InstanceId, + context.CancellationToken); + + await foreach (HistoryEvent historyEvent in historyEvents) + { + (historyChunk, currentChunkSizeInBytes) = await AddToHistoryChunkAndStream(historyEvent, historyChunk, currentChunkSizeInBytes); + } + } + + // Otherwise default to the older non-streaming implementation. + catch (NotImplementedException) + { + string jsonHistory = await this.GetDurabilityProvider(context).GetOrchestrationHistoryAsync( + request.InstanceId, + executionId: null); + + List? historyEvents = JsonConvert.DeserializeObject>( + jsonHistory, + new JsonSerializerSettings() + { + Converters = { new HistoryEventJsonConverter() }, + }) + ?? throw new Exception($"Failed to deserialize orchestration history."); + + foreach (HistoryEvent historyEvent in historyEvents) + { + (historyChunk, currentChunkSizeInBytes) = await AddToHistoryChunkAndStream(historyEvent, historyChunk, currentChunkSizeInBytes); + } + } + + // Send the last chunk, which may be smaller than the maximum chunk size. + if (historyChunk.Events.Count > 0) + { + await responseStream.WriteAsync(historyChunk); + } + } + catch (OperationCanceledException) + { + throw new RpcException(new Status(StatusCode.Cancelled, $"Orchestration history streaming cancelled for instance {request.InstanceId}")); + } + catch (Exception ex) + { + throw new RpcException(new Status(StatusCode.Internal, $"Failed to stream orchestration history for instance {request.InstanceId}: {ex.Message}")); + } } private static P.TaskFailureDetails? GetFailureDetails(FailureDetails? failureDetails) diff --git a/test/Common/DurableTaskEndToEndTests.cs b/test/Common/DurableTaskEndToEndTests.cs index 14cae0450..d9b139ac9 100644 --- a/test/Common/DurableTaskEndToEndTests.cs +++ b/test/Common/DurableTaskEndToEndTests.cs @@ -5395,14 +5395,11 @@ public async Task OverridableStates_TerminalStatusesAlwaysReusable(bool extended string instanceId = instanceIdBase + "_" + terminalStatus; TestDurableClient client; - if (terminalStatus != OrchestrationRuntimeStatus.Failed) - { - client = await host.StartOrchestratorAsync(nameof(TestOrchestrations.Counter), initialValue, this.output, instanceId: instanceId); - } - else - { - client = await host.StartOrchestratorAsync(nameof(TestOrchestrations.ThrowOrchestrator), string.Empty, this.output, instanceId: instanceId); - } + client = await host.StartOrchestratorAsync( + terminalStatus == OrchestrationRuntimeStatus.Failed ? nameof(TestOrchestrations.ThrowOrchestrator) : nameof(TestOrchestrations.Counter), + terminalStatus == OrchestrationRuntimeStatus.Failed ? string.Empty : initialValue, + this.output, + instanceId: instanceId); await client.WaitForStartupAsync(this.output); DurableOrchestrationStatus status = null; diff --git a/test/e2e/Apps/BasicDotNetIsolated/HelloCities.cs b/test/e2e/Apps/BasicDotNetIsolated/HelloCities.cs index 67b30e399..30cee1a07 100644 --- a/test/e2e/Apps/BasicDotNetIsolated/HelloCities.cs +++ b/test/e2e/Apps/BasicDotNetIsolated/HelloCities.cs @@ -109,7 +109,7 @@ public static async Task StartOrchestration_DedupeStatuses( if (scheduledStartTime is not null) { - startOptions = startOptions with { StartAt = scheduledStartTime }; + startOptions = startOptions with { StartAt = scheduledStartTime }; } // Function input comes from the request content. diff --git a/test/e2e/Tests/Tests/DedupeStatusesTests.cs b/test/e2e/Tests/Tests/DedupeStatusesTests.cs index 1e5ed9161..3a04d5fe6 100644 --- a/test/e2e/Tests/Tests/DedupeStatusesTests.cs +++ b/test/e2e/Tests/Tests/DedupeStatusesTests.cs @@ -139,7 +139,7 @@ private static async Task StartAndWaitForStateWithDedupeSta HttpStatusCode expectedCode = HttpStatusCode.Accepted) { string queryString = $"?orchestrationName={orchestrationName}&instanceId={instanceId}&dedupeStatuses={JsonSerializer.Serialize(dedupeStatuses)}"; - + if (scheduledStartTime is not null) { queryString += $"&scheduledStartTime={scheduledStartTime:o}"; From c22d39fc028a4f410b6b55a99e6634f93c5296da Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Tue, 20 Jan 2026 18:12:03 -0800 Subject: [PATCH 07/43] fixing whitespace --- test/e2e/Apps/BasicNode/src/functions/HelloCities.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/Apps/BasicNode/src/functions/HelloCities.ts b/test/e2e/Apps/BasicNode/src/functions/HelloCities.ts index c9667ee3c..1199d4d54 100644 --- a/test/e2e/Apps/BasicNode/src/functions/HelloCities.ts +++ b/test/e2e/Apps/BasicNode/src/functions/HelloCities.ts @@ -33,7 +33,7 @@ const HelloCitiesHttpStartScheduled: HttpHandler = async (request: HttpRequest, const client = df.getClient(context); const body: unknown = await request.text(); - const instanceId: string = await client.startNew("HelloCities", { input: request.params.ScheduledStartTime }); + const instanceId: string = await client.startNew("HelloCities", { input: request.params.ScheduledStartTime }); context.log(`Started orchestration with ID = '${instanceId}'.`); From 73e34ba19ca480ec9643a87972a00af76c19a6c3 Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Mon, 26 Jan 2026 11:49:18 -0800 Subject: [PATCH 08/43] removed linked cancellation token use --- .../DurabilityProvider.cs | 23 ++++++-------- .../DurableTaskExtension.cs | 2 -- .../HttpApiHandler.cs | 31 ++++++++++--------- .../Options/DurableTaskOptions.cs | 24 +------------- .../TaskHubGrpcServer.cs | 7 +---- 5 files changed, 28 insertions(+), 59 deletions(-) diff --git a/src/WebJobs.Extensions.DurableTask/DurabilityProvider.cs b/src/WebJobs.Extensions.DurableTask/DurabilityProvider.cs index 2f4645059..ced2d83dd 100644 --- a/src/WebJobs.Extensions.DurableTask/DurabilityProvider.cs +++ b/src/WebJobs.Extensions.DurableTask/DurabilityProvider.cs @@ -108,13 +108,12 @@ public DurabilityProvider(string storageProviderName, IOrchestrationService serv /// /// Event source name (e.g. DurableTask-AzureStorage). /// - public virtual string EventSourceName { get; set; } - + public virtual string EventSourceName { get; set; } + /// - /// Gets or sets the amount of time in seconds before a creation request for an orchestration times out. + /// Gets the amount of time in seconds before a creation request for an orchestration times out. /// Default value is 180 seconds. - /// - internal int OrchestrationCreationRequestTimeoutInSeconds { get; set; } = 180; + internal int OrchestrationCreationRequestTimeoutInSeconds { get; private set; } = 180; /// public int TaskOrchestrationDispatcherCount => this.GetOrchestrationService().TaskOrchestrationDispatcherCount; @@ -417,7 +416,7 @@ public Task CreateTaskOrchestrationAsync(TaskMessage creationMessage) /// /// should this method comment be updated too? to mention terminating existing instances? public async virtual Task CreateTaskOrchestrationAsync(TaskMessage creationMessage, OrchestrationStatus[] dedupeStatuses) - { + { using var timeoutCts = new CancellationTokenSource(TimeSpan.FromSeconds(this.OrchestrationCreationRequestTimeoutInSeconds)); await this.TerminateTaskOrchestrationWithReusableRunningStatusAndWaitAsync( creationMessage.OrchestrationInstance.InstanceId, @@ -437,22 +436,18 @@ await this.TerminateTaskOrchestrationWithReusableRunningStatusAndWaitAsync( /// and ), then only terminal statuses can be reused. /// If at least one of these statuses is not included in the array, then if an instance with that status is found, it will first be terminated /// before a new orchestration is created. - /// The cancellation token. + /// The cancellation token used to cancel waiting for an existing instance to terminate in the case that a + /// non-terminal instance is found whose runtime status is not included in . /// A task that completes when the creation message for the task orchestration instance is enqueued. /// Thrown if an orchestration with the same instance ID already exists and its status /// is in . /// Thrown if the operation is cancelled via . public async Task CreateTaskOrchestrationAsync(TaskMessage creationMessage, OrchestrationStatus[] dedupeStatuses, CancellationToken cancellationToken) - { - using var timeoutCts = new CancellationTokenSource(TimeSpan.FromSeconds(this.OrchestrationCreationRequestTimeoutInSeconds)); - using var linkedCts = - CancellationTokenSource.CreateLinkedTokenSource( - cancellationToken, - timeoutCts.Token); + { await this.TerminateTaskOrchestrationWithReusableRunningStatusAndWaitAsync( creationMessage.OrchestrationInstance.InstanceId, dedupeStatuses, - linkedCts.Token); + cancellationToken); await this.GetOrchestrationServiceClient().CreateTaskOrchestrationAsync(creationMessage, dedupeStatuses); } diff --git a/src/WebJobs.Extensions.DurableTask/DurableTaskExtension.cs b/src/WebJobs.Extensions.DurableTask/DurableTaskExtension.cs index 17c507333..2955f2440 100644 --- a/src/WebJobs.Extensions.DurableTask/DurableTaskExtension.cs +++ b/src/WebJobs.Extensions.DurableTask/DurableTaskExtension.cs @@ -183,8 +183,6 @@ public DurableTaskExtension( { this.OutOfProcProtocol = OutOfProcOrchestrationProtocol.OrchestratorShim; } - - this.defaultDurabilityProvider.OrchestrationCreationRequestTimeoutInSeconds = this.Options.OrchestrationCreationRequestTimeoutInSeconds; } internal DurableTaskOptions Options { get; } diff --git a/src/WebJobs.Extensions.DurableTask/HttpApiHandler.cs b/src/WebJobs.Extensions.DurableTask/HttpApiHandler.cs index b111ece08..2606cd039 100644 --- a/src/WebJobs.Extensions.DurableTask/HttpApiHandler.cs +++ b/src/WebJobs.Extensions.DurableTask/HttpApiHandler.cs @@ -934,13 +934,23 @@ private async Task HandleStartOrchestratorRequestAsync( using Activity scheduleOrchestrationActivity = TraceHelper.StartActivityForNewOrchestration(executionStartedEvent, default); } - await durableClient.DurabilityProvider.CreateTaskOrchestrationAsync( - new TaskMessage - { - Event = executionStartedEvent, - OrchestrationInstance = instance, - }, - this.config.Options.OverridableExistingInstanceStates.ToDedupeStatuses()); + try + { + await durableClient.DurabilityProvider.CreateTaskOrchestrationAsync( + new TaskMessage + { + Event = executionStartedEvent, + OrchestrationInstance = instance, + }, + this.config.Options.OverridableExistingInstanceStates.ToDedupeStatuses()); + } + catch (OperationCanceledException) + { + return request.CreateErrorResponse( + HttpStatusCode.RequestTimeout, + $"Create instance request exceeded timeout of {durableClient.DurabilityProvider.OrchestrationCreationRequestTimeoutInSeconds} " + + $"seconds for instance ID {instanceId} while waiting for the termination of the existing instance with this instance ID."); + } } else { @@ -973,13 +983,6 @@ await durableClient.DurabilityProvider.CreateTaskOrchestrationAsync( { return request.CreateErrorResponse(HttpStatusCode.Conflict, e.Message); } - catch (OperationCanceledException) - { - return request.CreateErrorResponse( - HttpStatusCode.RequestTimeout, - $"Create instance request exceeded timeout of {this.durableTaskOptions.OrchestrationCreationRequestTimeoutInSeconds} " + - $"seconds for instance ID {instanceId} while waiting for the termination of the existing instance with this instance ID."); - } } private static string GetHeaderValueFromHeaders(string header, HttpRequestHeaders headers) diff --git a/src/WebJobs.Extensions.DurableTask/Options/DurableTaskOptions.cs b/src/WebJobs.Extensions.DurableTask/Options/DurableTaskOptions.cs index d06570c1b..552312c5d 100644 --- a/src/WebJobs.Extensions.DurableTask/Options/DurableTaskOptions.cs +++ b/src/WebJobs.Extensions.DurableTask/Options/DurableTaskOptions.cs @@ -271,22 +271,7 @@ public string HubName /// Default is 100 seconds. /// This settings only applies when .NET 6 or greater is used. /// - public TimeSpan? GrpcHttpClientTimeout { get; set; } = TimeSpan.FromSeconds(100); - - /// - /// Gets or sets the amount of time in seconds before a creation request for an orchestration times out. - /// Default value is 180 seconds. - /// - /// - /// This setting is applicable when is set to . - /// If an orchestration in a non-terminal state already exists with the instance ID passed to the creation request, then this - /// orchestration will be terminated before the new orchestration is created. This setting controls how long the extension will wait - /// for the orchestration to reach a status of before failing the creation request. - /// - /// - /// The number of seconds before a creation request for an orchestration times out. - /// - public int OrchestrationCreationRequestTimeoutInSeconds { get; set; } = 180; + public TimeSpan? GrpcHttpClientTimeout { get; set; } = TimeSpan.FromSeconds(100); /// /// Gets or sets the local gRPC listener mode, controlling what version of gRPC listener is created. @@ -404,13 +389,6 @@ internal void Validate(INameResolver environmentVariableResolver) { throw new InvalidOperationException($"{nameof(this.MaxEntityOperationBatchSize)} must be a positive integer value."); } - - if (this.OrchestrationCreationRequestTimeoutInSeconds <= 0 || this.OrchestrationCreationRequestTimeoutInSeconds >= 230) - { - throw new InvalidOperationException($"{nameof(this.OrchestrationCreationRequestTimeoutInSeconds)} must be a positive integer value less than 230 seconds," + - $"which is the maximum amount of time that an HTTP triggered Function can take to respond to a request." + - $"See https://docs.azure.cn/en-us//azure-functions/functions-scale#timeout"); - } } internal bool IsDefaultHubName() diff --git a/src/WebJobs.Extensions.DurableTask/TaskHubGrpcServer.cs b/src/WebJobs.Extensions.DurableTask/TaskHubGrpcServer.cs index 8a23db56c..6d5797bc1 100644 --- a/src/WebJobs.Extensions.DurableTask/TaskHubGrpcServer.cs +++ b/src/WebJobs.Extensions.DurableTask/TaskHubGrpcServer.cs @@ -126,12 +126,7 @@ await this.GetDurabilityProvider(context).CreateTaskOrchestrationAsync( } catch (OperationCanceledException) { - throw new RpcException(new Status( - StatusCode.Cancelled, - context.CancellationToken.IsCancellationRequested - ? $"Create instance request cancelled for instance ID {request.InstanceId}" - : $"Create instance request exceeded timeout of {this.extension.Options.OrchestrationCreationRequestTimeoutInSeconds} seconds " + - $"for instance ID {request.InstanceId} while waiting for the termination of the existing instance with this instance ID.")); + throw new RpcException(new Status(StatusCode.Cancelled, $"Create instance request cancelled for instance ID {request.InstanceId}")); } catch (Exception ex) { From a90d03e29c720107dcab5569944608645a71609f Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Mon, 26 Jan 2026 12:23:59 -0800 Subject: [PATCH 09/43] updated the exception handling in the e2e test to only catch OrchestrationAlreadyExistsException --- test/e2e/Apps/BasicDotNetIsolated/HelloCities.cs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test/e2e/Apps/BasicDotNetIsolated/HelloCities.cs b/test/e2e/Apps/BasicDotNetIsolated/HelloCities.cs index 30cee1a07..0c8c08f74 100644 --- a/test/e2e/Apps/BasicDotNetIsolated/HelloCities.cs +++ b/test/e2e/Apps/BasicDotNetIsolated/HelloCities.cs @@ -117,9 +117,12 @@ public static async Task StartOrchestration_DedupeStatuses( { await client.ScheduleNewOrchestrationInstanceAsync(orchestrationName, startOptions); } - catch (Exception) + catch (OrchestrationAlreadyExistsException ex) { - return req.CreateResponse(HttpStatusCode.BadRequest); + // Tests expect BadRequest for orchestration dedupe scenarios. + HttpResponseData response = req.CreateResponse(HttpStatusCode.BadRequest); + await response.WriteStringAsync(ex.Message); + return response; } logger.LogInformation("Started orchestration with ID = '{instanceId}'.", instanceId); From ca2d617c29af544b036f951903de43399e27918b Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Mon, 26 Jan 2026 12:30:06 -0800 Subject: [PATCH 10/43] removing question comments --- src/WebJobs.Extensions.DurableTask/DurabilityProvider.cs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/WebJobs.Extensions.DurableTask/DurabilityProvider.cs b/src/WebJobs.Extensions.DurableTask/DurabilityProvider.cs index 28141f461..01fbb2fec 100644 --- a/src/WebJobs.Extensions.DurableTask/DurabilityProvider.cs +++ b/src/WebJobs.Extensions.DurableTask/DurabilityProvider.cs @@ -414,7 +414,6 @@ public Task CreateTaskOrchestrationAsync(TaskMessage creationMessage) } /// - /// should this method comment be updated too? to mention terminating existing instances? public async virtual Task CreateTaskOrchestrationAsync(TaskMessage creationMessage, OrchestrationStatus[] dedupeStatuses) { using var timeoutCts = new CancellationTokenSource(TimeSpan.FromSeconds(this.OrchestrationCreationRequestTimeoutInSeconds)); @@ -716,10 +715,6 @@ await this.ForceTerminateTaskOrchestrationAsync( await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken); orchestrationState = await this.GetOrchestrationStateAsync(instanceId, executionId: null); } - - // What should we do here? If dedupe statuses contains terminated, then the creation call afterwards will fail. - // Or should we throw an invalid argument exception if dedupeStatuses contains terminated but also allows for reuse of a running status? - // dedupeStatuses = dedupeStatuses.Except(new List() { OrchestrationStatus.Terminated }).ToArray(); } } } From ab99f05e3e5fafb892df903d64cae5fedbabf024 Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Mon, 26 Jan 2026 12:31:52 -0800 Subject: [PATCH 11/43] removed unnecessary usings --- src/WebJobs.Extensions.DurableTask/Options/DurableTaskOptions.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/WebJobs.Extensions.DurableTask/Options/DurableTaskOptions.cs b/src/WebJobs.Extensions.DurableTask/Options/DurableTaskOptions.cs index 552312c5d..fcf079735 100644 --- a/src/WebJobs.Extensions.DurableTask/Options/DurableTaskOptions.cs +++ b/src/WebJobs.Extensions.DurableTask/Options/DurableTaskOptions.cs @@ -6,7 +6,6 @@ using System.ComponentModel; using System.Net.Http; using DurableTask.AzureStorage.Partitioning; -using DurableTask.Core; using DurableTask.Core.Settings; using Microsoft.Azure.WebJobs.Extensions.DurableTask.Grpc; using Microsoft.Azure.WebJobs.Host; From 4aaccc266a138cd5a9a3d15a5642d53c5128fcc8 Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Mon, 26 Jan 2026 13:12:02 -0800 Subject: [PATCH 12/43] fixing a comment --- src/WebJobs.Extensions.DurableTask/DurabilityProvider.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/WebJobs.Extensions.DurableTask/DurabilityProvider.cs b/src/WebJobs.Extensions.DurableTask/DurabilityProvider.cs index 01fbb2fec..bec674edb 100644 --- a/src/WebJobs.Extensions.DurableTask/DurabilityProvider.cs +++ b/src/WebJobs.Extensions.DurableTask/DurabilityProvider.cs @@ -113,6 +113,7 @@ public DurabilityProvider(string storageProviderName, IOrchestrationService serv /// /// Gets the amount of time in seconds before a creation request for an orchestration times out. /// Default value is 180 seconds. + /// internal int OrchestrationCreationRequestTimeoutInSeconds { get; private set; } = 180; /// From 6d5ee8c5a97a0087d047777ed2acd9d704c645d8 Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Mon, 26 Jan 2026 14:38:25 -0800 Subject: [PATCH 13/43] updating the terminating poll to end if the orchestration reaches any terminal status --- src/WebJobs.Extensions.DurableTask/DurabilityProvider.cs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/WebJobs.Extensions.DurableTask/DurabilityProvider.cs b/src/WebJobs.Extensions.DurableTask/DurabilityProvider.cs index bec674edb..27f24fef2 100644 --- a/src/WebJobs.Extensions.DurableTask/DurabilityProvider.cs +++ b/src/WebJobs.Extensions.DurableTask/DurabilityProvider.cs @@ -685,6 +685,9 @@ private async Task TerminateTaskOrchestrationWithReusableRunningStatusAndWaitAsy OrchestrationStatus.Suspended, }; + bool IsRunning(OrchestrationStatus status) => + runningStatuses.Contains(status); + // At least one running status is reusable, so determine if an orchestration already exists with this status and terminate it if so if (runningStatuses.Any(status => !dedupeStatuses.Contains(status))) { @@ -698,7 +701,7 @@ private async Task TerminateTaskOrchestrationWithReusableRunningStatusAndWaitAsy $"'{orchestrationState.OrchestrationStatus}' already exists"); } - if (runningStatuses.Contains(orchestrationState.OrchestrationStatus)) + if (IsRunning(orchestrationState.OrchestrationStatus)) { // Check for cancellation before attempting to terminate the orchestration cancellationToken.ThrowIfCancellationRequested(); @@ -710,7 +713,7 @@ await this.ForceTerminateTaskOrchestrationAsync( $"{string.Join(", ", dedupeStatuses)}, do not contain the orchestration's status, the orchestration has been " + $"terminated and a new instance with the same instance ID will be created."); - while (orchestrationState != null && orchestrationState.OrchestrationStatus != OrchestrationStatus.Terminated) + while (orchestrationState != null && IsRunning(orchestrationState.OrchestrationStatus)) { cancellationToken.ThrowIfCancellationRequested(); await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken); From ff08413657586256881439d5be06027a917c1997 Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Mon, 26 Jan 2026 20:23:13 -0800 Subject: [PATCH 14/43] updated HTTP request to include cancellation token --- .../DurabilityProvider.cs | 2 +- .../HttpApiHandler.cs | 15 +++++++-------- .../LocalHttpListener.cs | 6 +++--- 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/src/WebJobs.Extensions.DurableTask/DurabilityProvider.cs b/src/WebJobs.Extensions.DurableTask/DurabilityProvider.cs index 27f24fef2..f5da91442 100644 --- a/src/WebJobs.Extensions.DurableTask/DurabilityProvider.cs +++ b/src/WebJobs.Extensions.DurableTask/DurabilityProvider.cs @@ -442,7 +442,7 @@ await this.TerminateTaskOrchestrationWithReusableRunningStatusAndWaitAsync( /// Thrown if an orchestration with the same instance ID already exists and its status /// is in . /// Thrown if the operation is cancelled via . - public async Task CreateTaskOrchestrationAsync(TaskMessage creationMessage, OrchestrationStatus[] dedupeStatuses, CancellationToken cancellationToken) + public async virtual Task CreateTaskOrchestrationAsync(TaskMessage creationMessage, OrchestrationStatus[] dedupeStatuses, CancellationToken cancellationToken) { await this.TerminateTaskOrchestrationWithReusableRunningStatusAndWaitAsync( creationMessage.OrchestrationInstance.InstanceId, diff --git a/src/WebJobs.Extensions.DurableTask/HttpApiHandler.cs b/src/WebJobs.Extensions.DurableTask/HttpApiHandler.cs index 2606cd039..39e1a4d1f 100644 --- a/src/WebJobs.Extensions.DurableTask/HttpApiHandler.cs +++ b/src/WebJobs.Extensions.DurableTask/HttpApiHandler.cs @@ -283,7 +283,7 @@ internal async Task WaitForCompletionOrCreateCheckStatusRes } } - public async Task HandleRequestAsync(HttpRequestMessage request) + public async Task HandleRequestAsync(HttpRequestMessage request, CancellationToken cancellationToken) { try { @@ -307,7 +307,7 @@ public async Task HandleRequestAsync(HttpRequestMessage req string instanceId = (string)routeValues[InstanceIdRouteParameter]; if (request.Method == HttpMethod.Post) { - return await this.HandleStartOrchestratorRequestAsync(request, functionName, instanceId); + return await this.HandleStartOrchestratorRequestAsync(request, functionName, instanceId, cancellationToken); } else { @@ -884,7 +884,8 @@ private async Task HandleResumeInstanceRequestAsync( private async Task HandleStartOrchestratorRequestAsync( HttpRequestMessage request, string functionName, - string instanceId) + string instanceId, + CancellationToken cancellationToken) { try { @@ -942,14 +943,12 @@ await durableClient.DurabilityProvider.CreateTaskOrchestrationAsync( Event = executionStartedEvent, OrchestrationInstance = instance, }, - this.config.Options.OverridableExistingInstanceStates.ToDedupeStatuses()); + this.config.Options.OverridableExistingInstanceStates.ToDedupeStatuses(), + cancellationToken); } catch (OperationCanceledException) { - return request.CreateErrorResponse( - HttpStatusCode.RequestTimeout, - $"Create instance request exceeded timeout of {durableClient.DurabilityProvider.OrchestrationCreationRequestTimeoutInSeconds} " + - $"seconds for instance ID {instanceId} while waiting for the termination of the existing instance with this instance ID."); + return request.CreateErrorResponse(HttpStatusCode.RequestTimeout, $"Create instance request cancelled for instance ID {instanceId}"); } } else diff --git a/src/WebJobs.Extensions.DurableTask/LocalHttpListener.cs b/src/WebJobs.Extensions.DurableTask/LocalHttpListener.cs index 2d0c4662f..a1eac2d4c 100644 --- a/src/WebJobs.Extensions.DurableTask/LocalHttpListener.cs +++ b/src/WebJobs.Extensions.DurableTask/LocalHttpListener.cs @@ -28,7 +28,7 @@ internal class LocalHttpListener : IDisposable private const int MinPort = 30000; private const int MaxPort = 31000; - private readonly Func> handler; + private readonly Func> handler; private readonly EndToEndTraceHelper traceHelper; private readonly DurableTaskOptions durableTaskOptions; private readonly Random portGenerator; @@ -39,7 +39,7 @@ internal class LocalHttpListener : IDisposable public LocalHttpListener( EndToEndTraceHelper traceHelper, DurableTaskOptions durableTaskOptions, - Func> handler) + Func> handler) { this.traceHelper = traceHelper ?? throw new ArgumentNullException(nameof(traceHelper)); this.handler = handler ?? throw new ArgumentNullException(nameof(handler)); @@ -135,7 +135,7 @@ private async Task HandleRequestAsync(HttpContext context) try { HttpRequestMessage request = GetRequest(context); - HttpResponseMessage response = await this.handler(request); + HttpResponseMessage response = await this.handler(request, context.RequestAborted); await SetResponseAsync(context, response); } catch (Exception e) From ff3022079759b472b4e285090a573463f680faa3 Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Tue, 27 Jan 2026 11:41:01 -0800 Subject: [PATCH 15/43] fixing the build bug --- src/WebJobs.Extensions.DurableTask/DurableTaskExtension.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/WebJobs.Extensions.DurableTask/DurableTaskExtension.cs b/src/WebJobs.Extensions.DurableTask/DurableTaskExtension.cs index 3083a2111..7900c4282 100644 --- a/src/WebJobs.Extensions.DurableTask/DurableTaskExtension.cs +++ b/src/WebJobs.Extensions.DurableTask/DurableTaskExtension.cs @@ -1471,7 +1471,7 @@ Task IAsyncConverter Date: Tue, 27 Jan 2026 12:31:07 -0800 Subject: [PATCH 16/43] fixed CreateTaskOrchestration overrides to ultimately call the same method --- .../DurabilityProvider.cs | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/src/WebJobs.Extensions.DurableTask/DurabilityProvider.cs b/src/WebJobs.Extensions.DurableTask/DurabilityProvider.cs index f5da91442..e779516fd 100644 --- a/src/WebJobs.Extensions.DurableTask/DurabilityProvider.cs +++ b/src/WebJobs.Extensions.DurableTask/DurabilityProvider.cs @@ -40,6 +40,7 @@ public class DurabilityProvider : private readonly IOrchestrationServiceClient innerServiceClient; private readonly IEntityOrchestrationService entityOrchestrationService; private readonly string connectionName; + private readonly int orchestrationCreationRequestTimeoutInSeconds = 180; /// /// Creates the default . @@ -110,12 +111,6 @@ public DurabilityProvider(string storageProviderName, IOrchestrationService serv /// public virtual string EventSourceName { get; set; } - /// - /// Gets the amount of time in seconds before a creation request for an orchestration times out. - /// Default value is 180 seconds. - /// - internal int OrchestrationCreationRequestTimeoutInSeconds { get; private set; } = 180; - /// public int TaskOrchestrationDispatcherCount => this.GetOrchestrationService().TaskOrchestrationDispatcherCount; @@ -415,14 +410,10 @@ public Task CreateTaskOrchestrationAsync(TaskMessage creationMessage) } /// - public async virtual Task CreateTaskOrchestrationAsync(TaskMessage creationMessage, OrchestrationStatus[] dedupeStatuses) + public Task CreateTaskOrchestrationAsync(TaskMessage creationMessage, OrchestrationStatus[] dedupeStatuses) { - using var timeoutCts = new CancellationTokenSource(TimeSpan.FromSeconds(this.OrchestrationCreationRequestTimeoutInSeconds)); - await this.TerminateTaskOrchestrationWithReusableRunningStatusAndWaitAsync( - creationMessage.OrchestrationInstance.InstanceId, - dedupeStatuses, - timeoutCts.Token); - await this.GetOrchestrationServiceClient().CreateTaskOrchestrationAsync(creationMessage, dedupeStatuses); + using var timeoutCts = new CancellationTokenSource(TimeSpan.FromSeconds(this.orchestrationCreationRequestTimeoutInSeconds)); + return this.CreateTaskOrchestrationAsync(creationMessage, dedupeStatuses, timeoutCts.Token); } /// From aad600297c7f3adc41c5e39207e3ae07ad17d9fa Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Wed, 4 Feb 2026 15:24:35 -0800 Subject: [PATCH 17/43] PR comments: --- .../HttpApiHandler.cs | 23 +++++++------------ .../TaskHubGrpcServer.cs | 18 ++++----------- test/Common/DurableTaskEndToEndTests.cs | 4 ++++ 3 files changed, 16 insertions(+), 29 deletions(-) diff --git a/src/WebJobs.Extensions.DurableTask/HttpApiHandler.cs b/src/WebJobs.Extensions.DurableTask/HttpApiHandler.cs index 39e1a4d1f..a1d839711 100644 --- a/src/WebJobs.Extensions.DurableTask/HttpApiHandler.cs +++ b/src/WebJobs.Extensions.DurableTask/HttpApiHandler.cs @@ -935,21 +935,14 @@ private async Task HandleStartOrchestratorRequestAsync( using Activity scheduleOrchestrationActivity = TraceHelper.StartActivityForNewOrchestration(executionStartedEvent, default); } - try - { - await durableClient.DurabilityProvider.CreateTaskOrchestrationAsync( - new TaskMessage - { - Event = executionStartedEvent, - OrchestrationInstance = instance, - }, - this.config.Options.OverridableExistingInstanceStates.ToDedupeStatuses(), - cancellationToken); - } - catch (OperationCanceledException) - { - return request.CreateErrorResponse(HttpStatusCode.RequestTimeout, $"Create instance request cancelled for instance ID {instanceId}"); - } + await durableClient.DurabilityProvider.CreateTaskOrchestrationAsync( + new TaskMessage + { + Event = executionStartedEvent, + OrchestrationInstance = instance, + }, + this.config.Options.OverridableExistingInstanceStates.ToDedupeStatuses(), + cancellationToken); } else { diff --git a/src/WebJobs.Extensions.DurableTask/TaskHubGrpcServer.cs b/src/WebJobs.Extensions.DurableTask/TaskHubGrpcServer.cs index 6d5797bc1..3a4c13b30 100644 --- a/src/WebJobs.Extensions.DurableTask/TaskHubGrpcServer.cs +++ b/src/WebJobs.Extensions.DurableTask/TaskHubGrpcServer.cs @@ -53,16 +53,10 @@ public override Task Hello(Empty request, ServerCallContext context) public async override Task StartInstance(P.CreateInstanceRequest request, ServerCallContext context) { try - { - var allStatuses = new List() - { - OrchestrationStatus.Running, - OrchestrationStatus.Pending, - OrchestrationStatus.Suspended, - OrchestrationStatus.Completed, - OrchestrationStatus.Failed, - OrchestrationStatus.Terminated, - }; + { + List allStatuses = System.Enum + .GetValues() + .ToList(); // Not all clients are necessarily configured to set the OrchestrationIdReusePolicy field of the request. // If it is null, we assume that they do not support per-request-dedupe statuses, and default to using just @@ -123,10 +117,6 @@ await this.GetDurabilityProvider(context).CreateTaskOrchestrationAsync( catch (InvalidOperationException ex) when (ex.Message.EndsWith("already exists.")) // for older versions of DTF.AS and DTFx.Netherite { throw new RpcException(new Status(StatusCode.AlreadyExists, $"An Orchestration instance with the ID {request.InstanceId} already exists.")); - } - catch (OperationCanceledException) - { - throw new RpcException(new Status(StatusCode.Cancelled, $"Create instance request cancelled for instance ID {request.InstanceId}")); } catch (Exception ex) { diff --git a/test/Common/DurableTaskEndToEndTests.cs b/test/Common/DurableTaskEndToEndTests.cs index bde556984..6dc7f8e2d 100644 --- a/test/Common/DurableTaskEndToEndTests.cs +++ b/test/Common/DurableTaskEndToEndTests.cs @@ -5279,6 +5279,8 @@ await Assert.ThrowsAsync(async () => } } + // This method returns an array of [bool extendedSessions, string storageProvider, bool anyStateOverridable, bool suspend] + // It combines the GetBooleanAndFullFeaturedStorageProviderOptionsWithOverridableStates with both true and false for suspend. public static IEnumerable GetBooleanAndFullFeaturedStorageProviderOptionsWithOverridableStatesAndSuspend() { foreach (object[] data in GetBooleanAndFullFeaturedStorageProviderOptionsWithOverridableStates()) @@ -5288,6 +5290,8 @@ public static IEnumerable GetBooleanAndFullFeaturedStorageProviderOpti } } + // This method returns an array of [bool extendedSessions, string storageProvider, bool anyStateOverridable] + // It combines the TestDataGenerator.GetBooleanAndFullFeaturedStorageProviderOptions with both true and false for anyStateOverridable. public static IEnumerable GetBooleanAndFullFeaturedStorageProviderOptionsWithOverridableStates() { foreach (object[] data in TestDataGenerator.GetBooleanAndFullFeaturedStorageProviderOptions()) From 33d0f630bdd204aa7996ced35ee3452956772631 Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Wed, 4 Feb 2026 22:49:11 -0800 Subject: [PATCH 18/43] updating the termination poll logic to use WaitForOrchestration --- .../DurabilityProvider.cs | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/WebJobs.Extensions.DurableTask/DurabilityProvider.cs b/src/WebJobs.Extensions.DurableTask/DurabilityProvider.cs index e779516fd..74674befd 100644 --- a/src/WebJobs.Extensions.DurableTask/DurabilityProvider.cs +++ b/src/WebJobs.Extensions.DurableTask/DurabilityProvider.cs @@ -704,12 +704,11 @@ await this.ForceTerminateTaskOrchestrationAsync( $"{string.Join(", ", dedupeStatuses)}, do not contain the orchestration's status, the orchestration has been " + $"terminated and a new instance with the same instance ID will be created."); - while (orchestrationState != null && IsRunning(orchestrationState.OrchestrationStatus)) - { - cancellationToken.ThrowIfCancellationRequested(); - await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken); - orchestrationState = await this.GetOrchestrationStateAsync(instanceId, executionId: null); - } + await this.WaitForOrchestrationAsync( + instanceId, + orchestrationState.OrchestrationInstance.ExecutionId, + TimeSpan.MaxValue, + cancellationToken); } } } From 0f0f28d632edb51504d72ed5fecd1e71303fc3c2 Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Tue, 10 Feb 2026 18:16:56 -0800 Subject: [PATCH 19/43] Adding an ArgumentException for invalid dedupe statuses (any running + terminated) --- .../DurabilityProvider.cs | 45 ++++++++- .../HttpApiHandler.cs | 5 + .../TaskHubGrpcServer.cs | 4 + .../Apps/BasicDotNetIsolated/HelloCities.cs | 7 ++ test/e2e/Tests/Tests/DedupeStatusesTests.cs | 98 ++++++++++++++----- 5 files changed, 127 insertions(+), 32 deletions(-) diff --git a/src/WebJobs.Extensions.DurableTask/DurabilityProvider.cs b/src/WebJobs.Extensions.DurableTask/DurabilityProvider.cs index 74674befd..bda44a546 100644 --- a/src/WebJobs.Extensions.DurableTask/DurabilityProvider.cs +++ b/src/WebJobs.Extensions.DurableTask/DurabilityProvider.cs @@ -409,7 +409,27 @@ public Task CreateTaskOrchestrationAsync(TaskMessage creationMessage) return this.GetOrchestrationServiceClient().CreateTaskOrchestrationAsync(creationMessage); } - /// + /// + /// Creates a new task orchestration instance using the specified creation message and dedupe statuses. + /// + /// The creation message for the orchestration. + /// An array of orchestration statuses used for "dedupping": + /// If an orchestration with the same instance ID already exists, and its status is in this array, then a + /// will be thrown. + /// If the array contains all of the running statuses (, , + /// and ), then only terminal statuses can be reused. + /// If at least one of these statuses is not included in the array, then if an instance with that status is found, it will first be terminated + /// before a new orchestration is created. If the existing instance does not reach a terminal state within 3 minutes, the operation will be cancelled. + /// + /// A task that completes when the creation message for the task orchestration instance is enqueued. + /// Thrown if an orchestration with the same instance ID already exists and its status + /// is in . + /// Thrown if an existing running instance does not reach a terminal state within 3 minutes. + /// + /// Thrown if contains but also allows at least one running status + /// to be reusable. In this case, an existing orchestration with that running status would be terminated, but the creation of the new orchestration + /// would immediately fail due to the existing orchestration now having status . + /// public Task CreateTaskOrchestrationAsync(TaskMessage creationMessage, OrchestrationStatus[] dedupeStatuses) { using var timeoutCts = new CancellationTokenSource(TimeSpan.FromSeconds(this.orchestrationCreationRequestTimeoutInSeconds)); @@ -433,6 +453,11 @@ public Task CreateTaskOrchestrationAsync(TaskMessage creationMessage, Orchestrat /// Thrown if an orchestration with the same instance ID already exists and its status /// is in . /// Thrown if the operation is cancelled via . + /// + /// Thrown if contains but also allows at least one running status + /// to be reusable. In this case, an existing orchestration with that running status would be terminated, but the creation of the new orchestration + /// would immediately fail due to the existing orchestration now having status . + /// public async virtual Task CreateTaskOrchestrationAsync(TaskMessage creationMessage, OrchestrationStatus[] dedupeStatuses, CancellationToken cancellationToken) { await this.TerminateTaskOrchestrationWithReusableRunningStatusAndWaitAsync( @@ -664,6 +689,8 @@ public virtual void SetUseSeparateQueueForEntityWorkItems(bool newValue) /// A task that completes when any of the above conditions are reached. /// Thrown if the operation is cancelled via the . /// Thrown if an orchestration already exists with status in . + /// Thrown if contains but allows + /// at least one running status to be reusable. private async Task TerminateTaskOrchestrationWithReusableRunningStatusAndWaitAsync( string instanceId, OrchestrationStatus[] dedupeStatuses, @@ -676,17 +703,25 @@ private async Task TerminateTaskOrchestrationWithReusableRunningStatusAndWaitAsy OrchestrationStatus.Suspended, }; + if (dedupeStatuses != null && runningStatuses.Any( + status => !dedupeStatuses.Contains(status)) && dedupeStatuses.Contains(OrchestrationStatus.Terminated)) + { + throw new ArgumentException( + "Invalid dedupe statuses: cannot include 'Terminated' while also allowing reuse of running instances, " + + "because the running instance would be terminated and then immediately conflict with the dedupe check."); + } + bool IsRunning(OrchestrationStatus status) => runningStatuses.Contains(status); // At least one running status is reusable, so determine if an orchestration already exists with this status and terminate it if so - if (runningStatuses.Any(status => !dedupeStatuses.Contains(status))) + if (dedupeStatuses == null || runningStatuses.Any(status => !dedupeStatuses.Contains(status))) { OrchestrationState orchestrationState = await this.GetOrchestrationStateAsync(instanceId, executionId: null); if (orchestrationState != null) { - if (dedupeStatuses.Contains(orchestrationState.OrchestrationStatus)) + if (dedupeStatuses?.Contains(orchestrationState.OrchestrationStatus) == true) { throw new OrchestrationAlreadyExistsException($"An orchestration with instance ID '{instanceId}' and status " + $"'{orchestrationState.OrchestrationStatus}' already exists"); @@ -701,8 +736,8 @@ await this.ForceTerminateTaskOrchestrationAsync( instanceId, $"A new instance creation request has been issued for instance {instanceId} which currently has status " + $"{orchestrationState.OrchestrationStatus}. Since the dedupe statuses of the creation request, " + - $"{string.Join(", ", dedupeStatuses)}, do not contain the orchestration's status, the orchestration has been " + - $"terminated and a new instance with the same instance ID will be created."); + $"{(dedupeStatuses == null ? "[]" : string.Join(", ", dedupeStatuses))}, do not contain the orchestration's " + + $"status, the orchestration has been terminated and a new instance with the same instance ID will be created."); await this.WaitForOrchestrationAsync( instanceId, diff --git a/src/WebJobs.Extensions.DurableTask/HttpApiHandler.cs b/src/WebJobs.Extensions.DurableTask/HttpApiHandler.cs index a1d839711..fba24b9a8 100644 --- a/src/WebJobs.Extensions.DurableTask/HttpApiHandler.cs +++ b/src/WebJobs.Extensions.DurableTask/HttpApiHandler.cs @@ -14,6 +14,7 @@ using DurableTask.Core; using DurableTask.Core.Exceptions; using DurableTask.Core.History; +using Grpc.Core; using Microsoft.AspNetCore.Routing; using Microsoft.AspNetCore.Routing.Template; using Microsoft.Azure.WebJobs.Extensions.DurableTask.Correlation; @@ -975,6 +976,10 @@ await durableClient.DurabilityProvider.CreateTaskOrchestrationAsync( { return request.CreateErrorResponse(HttpStatusCode.Conflict, e.Message); } + catch (ArgumentException e) + { + return request.CreateErrorResponse(HttpStatusCode.BadRequest, e.Message); + } } private static string GetHeaderValueFromHeaders(string header, HttpRequestHeaders headers) diff --git a/src/WebJobs.Extensions.DurableTask/TaskHubGrpcServer.cs b/src/WebJobs.Extensions.DurableTask/TaskHubGrpcServer.cs index 3a4c13b30..2e0541f5c 100644 --- a/src/WebJobs.Extensions.DurableTask/TaskHubGrpcServer.cs +++ b/src/WebJobs.Extensions.DurableTask/TaskHubGrpcServer.cs @@ -118,6 +118,10 @@ await this.GetDurabilityProvider(context).CreateTaskOrchestrationAsync( { throw new RpcException(new Status(StatusCode.AlreadyExists, $"An Orchestration instance with the ID {request.InstanceId} already exists.")); } + catch (ArgumentException ex) + { + throw new RpcException(new Status(StatusCode.InvalidArgument, $"Invalid argument for start instance request for instance ID {request.InstanceId}: {ex.Message}")); + } catch (Exception ex) { this.extension.TraceHelper.ExtensionWarningEvent( diff --git a/test/e2e/Apps/BasicDotNetIsolated/HelloCities.cs b/test/e2e/Apps/BasicDotNetIsolated/HelloCities.cs index 0c8c08f74..0b7900531 100644 --- a/test/e2e/Apps/BasicDotNetIsolated/HelloCities.cs +++ b/test/e2e/Apps/BasicDotNetIsolated/HelloCities.cs @@ -118,6 +118,13 @@ public static async Task StartOrchestration_DedupeStatuses( await client.ScheduleNewOrchestrationInstanceAsync(orchestrationName, startOptions); } catch (OrchestrationAlreadyExistsException ex) + { + // Tests expect BadRequest for orchestration dedupe scenarios. + HttpResponseData response = req.CreateResponse(HttpStatusCode.Conflict); + await response.WriteStringAsync(ex.Message); + return response; + } + catch (ArgumentException ex) { // Tests expect BadRequest for orchestration dedupe scenarios. HttpResponseData response = req.CreateResponse(HttpStatusCode.BadRequest); diff --git a/test/e2e/Tests/Tests/DedupeStatusesTests.cs b/test/e2e/Tests/Tests/DedupeStatusesTests.cs index 3a04d5fe6..d1699fdb9 100644 --- a/test/e2e/Tests/Tests/DedupeStatusesTests.cs +++ b/test/e2e/Tests/Tests/DedupeStatusesTests.cs @@ -23,46 +23,60 @@ public async Task CanStartOrchestrationWithSameIdForAllStatusesForEmptyDedupeSta { // Completed string completedInstanceId = Guid.NewGuid().ToString(); - using HttpResponseMessage startCompletedResponseFirstAttempt = await StartAndWaitForState("HelloCities", completedInstanceId, "Completed"); - using HttpResponseMessage startCompletedResponseSecondAttempt = await StartAndWaitForState("HelloCities", completedInstanceId, "Completed"); + using HttpResponseMessage startCompletedResponseFirstAttempt = await StartAndWaitForState( + "HelloCities", completedInstanceId, "Completed"); + using HttpResponseMessage startCompletedResponseSecondAttempt = await StartAndWaitForState( + "HelloCities", completedInstanceId, "Completed"); // Failed string failedInstanceId = Guid.NewGuid().ToString(); - using HttpResponseMessage startFailedResponseFirstAttempt = await StartAndWaitForState("RethrowActivityException", failedInstanceId, "Failed"); - // Invoking this same orchestration with the same instance ID will cause it to complete successfully on the second attempt, hence we look for a "Completed" status instead - using HttpResponseMessage startFailedResponseSecondAttempt = await StartAndWaitForState("RethrowActivityException", failedInstanceId, "Completed"); + using HttpResponseMessage startFailedResponseFirstAttempt = await StartAndWaitForState( + "RethrowActivityException", failedInstanceId, "Failed"); + // Invoking this same orchestration with the same instance ID will cause it to complete successfully on the second attempt, + // hence we look for a "Completed" status instead + using HttpResponseMessage startFailedResponseSecondAttempt = await StartAndWaitForState( + "RethrowActivityException", failedInstanceId, "Completed"); // Terminated if (this.fixture.functionLanguageLocalizer.GetLanguageType() != LanguageType.Java || this.fixture.GetDurabilityProvider() != FunctionAppFixture.ConfiguredDurabilityProviderType.MSSQL) // Bug: https://github.com/microsoft/durabletask-java/issues/237 { string terminatedInstanceId = Guid.NewGuid().ToString(); - using HttpResponseMessage startTerminatedResponseFirstAttempt = await StartAndWaitForState("LongRunningOrchestrator", terminatedInstanceId, "Running"); + using HttpResponseMessage startTerminatedResponseFirstAttempt = await StartAndWaitForState( + "LongRunningOrchestrator", terminatedInstanceId, "Running"); await TerminateAndWaitForState(terminatedInstanceId, startTerminatedResponseFirstAttempt); - using HttpResponseMessage startTerminatedResponseSecondAttempt = await StartAndWaitForState("LongRunningOrchestrator", terminatedInstanceId, "Running"); + using HttpResponseMessage startTerminatedResponseSecondAttempt = await StartAndWaitForState( + "LongRunningOrchestrator", terminatedInstanceId, "Running"); } // Pending - // Scheduled start times are currently only implemented in Java and .NET isolated, which is the only true way to get an orchestration in a "Pending" state + // Scheduled start times are currently only implemented in Java and .NET isolated, which is the only true way + // to get an orchestration in a "Pending" state if (this.fixture.functionLanguageLocalizer.GetLanguageType() == LanguageType.DotnetIsolated || this.fixture.functionLanguageLocalizer.GetLanguageType() == LanguageType.Java) { string pendingInstanceId = Guid.NewGuid().ToString(); DateTime scheduledStartTime = DateTime.UtcNow.AddMinutes(2); - using HttpResponseMessage startPendingResponseFirstAttempt = await StartAndWaitForState("HelloCities", pendingInstanceId, "Pending", scheduledStartTime: scheduledStartTime); - using HttpResponseMessage startPendingResponseSecondAttempt = await StartAndWaitForState("HelloCities", pendingInstanceId, "Completed"); + using HttpResponseMessage startPendingResponseFirstAttempt = await StartAndWaitForState( + "HelloCities", pendingInstanceId, "Pending", scheduledStartTime: scheduledStartTime); + using HttpResponseMessage startPendingResponseSecondAttempt = await StartAndWaitForState( + "HelloCities", pendingInstanceId, "Completed"); } // Running string runningInstanceId = Guid.NewGuid().ToString(); - using HttpResponseMessage startRunningResponseFirstAttempt = await StartAndWaitForState("LongRunningOrchestrator", runningInstanceId, "Running"); - using HttpResponseMessage startRunningResponseSecondAttempt = await StartAndWaitForState("LongRunningOrchestrator", runningInstanceId, "Running"); + using HttpResponseMessage startRunningResponseFirstAttempt = await StartAndWaitForState( + "LongRunningOrchestrator", runningInstanceId, "Running"); + using HttpResponseMessage startRunningResponseSecondAttempt = await StartAndWaitForState( + "LongRunningOrchestrator", runningInstanceId, "Running"); // Suspended string suspendedInstanceId = Guid.NewGuid().ToString(); - using HttpResponseMessage startSuspendedResponseFirstAttempt = await StartAndWaitForState("LongRunningOrchestrator", suspendedInstanceId, "Running"); + using HttpResponseMessage startSuspendedResponseFirstAttempt = await StartAndWaitForState( + "LongRunningOrchestrator", suspendedInstanceId, "Running"); await SuspendAndWaitForState(suspendedInstanceId, startSuspendedResponseFirstAttempt); - using HttpResponseMessage startSuspendedResponseSecondAttempt = await StartAndWaitForState("LongRunningOrchestrator", suspendedInstanceId, "Running"); + using HttpResponseMessage startSuspendedResponseSecondAttempt = await StartAndWaitForState( + "LongRunningOrchestrator", suspendedInstanceId, "Running"); } [Fact] @@ -76,36 +90,65 @@ public async Task StartOrchestrationWithSameIdFailsForDedupeStatuses() // Completed, should succeed string completedInstanceId = Guid.NewGuid().ToString(); - using HttpResponseMessage startCompletedResponseFirstAttempt = await StartAndWaitForStateWithDedupeStatuses("HelloCities", completedInstanceId, "Completed", dedupeStatuses); - using HttpResponseMessage startCompletedResponseSecondAttempt = await StartAndWaitForStateWithDedupeStatuses("HelloCities", completedInstanceId, "Completed", dedupeStatuses); + using HttpResponseMessage startCompletedResponseFirstAttempt = await StartAndWaitForStateWithDedupeStatuses( + "HelloCities", completedInstanceId, "Completed", dedupeStatuses); + using HttpResponseMessage startCompletedResponseSecondAttempt = await StartAndWaitForStateWithDedupeStatuses( + "HelloCities", completedInstanceId, "Completed", dedupeStatuses); // Terminated string terminatedInstanceId = Guid.NewGuid().ToString(); - using HttpResponseMessage startTerminatedResponseFirstAttempt = await StartAndWaitForStateWithDedupeStatuses("LongRunningOrchestrator", terminatedInstanceId, "Running", dedupeStatuses); + using HttpResponseMessage startTerminatedResponseFirstAttempt = await StartAndWaitForStateWithDedupeStatuses( + "LongRunningOrchestrator", terminatedInstanceId, "Running", dedupeStatuses); await TerminateAndWaitForState(terminatedInstanceId, startTerminatedResponseFirstAttempt); - using HttpResponseMessage startTerminatedResponseSecondAttempt = await StartAndWaitForStateWithDedupeStatuses("LongRunningOrchestrator", terminatedInstanceId, "Running", dedupeStatuses); + using HttpResponseMessage startTerminatedResponseSecondAttempt = await StartAndWaitForStateWithDedupeStatuses( + "LongRunningOrchestrator", terminatedInstanceId, "Running", dedupeStatuses); // Pending string pendingInstanceId = Guid.NewGuid().ToString(); DateTime scheduledStartTime = DateTime.UtcNow.AddMinutes(2); - using HttpResponseMessage startPendingResponseFirstAttempt = await StartAndWaitForStateWithDedupeStatuses("HelloCities", pendingInstanceId, "Pending", dedupeStatuses, scheduledStartTime: scheduledStartTime); - using HttpResponseMessage startPendingResponseSecondAttempt = await StartAndWaitForStateWithDedupeStatuses("HelloCities", pendingInstanceId, "Completed", dedupeStatuses); + using HttpResponseMessage startPendingResponseFirstAttempt = await StartAndWaitForStateWithDedupeStatuses( + "HelloCities", pendingInstanceId, "Pending", dedupeStatuses, scheduledStartTime: scheduledStartTime); + using HttpResponseMessage startPendingResponseSecondAttempt = await StartAndWaitForStateWithDedupeStatuses( + "HelloCities", pendingInstanceId, "Completed", dedupeStatuses); // Suspended string suspendedInstanceId = Guid.NewGuid().ToString(); - using HttpResponseMessage startSuspendedResponseFirstAttempt = await StartAndWaitForStateWithDedupeStatuses("LongRunningOrchestrator", suspendedInstanceId, "Running", dedupeStatuses); + using HttpResponseMessage startSuspendedResponseFirstAttempt = await StartAndWaitForStateWithDedupeStatuses( + "LongRunningOrchestrator", suspendedInstanceId, "Running", dedupeStatuses); await SuspendAndWaitForState(suspendedInstanceId, startSuspendedResponseFirstAttempt); - using HttpResponseMessage startSuspendedResponseSecondAttempt = await StartAndWaitForStateWithDedupeStatuses("LongRunningOrchestrator", suspendedInstanceId, "Running", dedupeStatuses); + using HttpResponseMessage startSuspendedResponseSecondAttempt = await StartAndWaitForStateWithDedupeStatuses( + "LongRunningOrchestrator", suspendedInstanceId, "Running", dedupeStatuses); // Failed, should fail string failedInstanceId = Guid.NewGuid().ToString(); - using HttpResponseMessage startFailedResponseFirstAttempt = await StartAndWaitForStateWithDedupeStatuses("RethrowActivityException", failedInstanceId, "Failed", dedupeStatuses); - using HttpResponseMessage startFailedResponseSecondAttempt = await StartAndWaitForStateWithDedupeStatuses("RethrowActivityException", failedInstanceId, "Failed", dedupeStatuses, expectedCode: HttpStatusCode.BadRequest); + using HttpResponseMessage startFailedResponseFirstAttempt = await StartAndWaitForStateWithDedupeStatuses( + "RethrowActivityException", failedInstanceId, "Failed", dedupeStatuses); + // We do not provide an expected state since we expect the request to fail + using HttpResponseMessage startFailedResponseSecondAttempt = await StartAndWaitForStateWithDedupeStatuses( + "RethrowActivityException", failedInstanceId, expectedState: string.Empty, dedupeStatuses, expectedCode: HttpStatusCode.Conflict); // Running, should fail string runningInstanceId = Guid.NewGuid().ToString(); - using HttpResponseMessage startRunningResponseFirstAttempt = await StartAndWaitForStateWithDedupeStatuses("LongRunningOrchestrator", runningInstanceId, "Running", dedupeStatuses); - using HttpResponseMessage startRunningResponseSecondAttempt = await StartAndWaitForStateWithDedupeStatuses("LongRunningOrchestrator", runningInstanceId, "Running", dedupeStatuses, expectedCode: HttpStatusCode.BadRequest); + using HttpResponseMessage startRunningResponseFirstAttempt = await StartAndWaitForStateWithDedupeStatuses( + "LongRunningOrchestrator", runningInstanceId, "Running", dedupeStatuses); + // We do not provide an expected state since we expect the request to fail + using HttpResponseMessage startRunningResponseSecondAttempt = await StartAndWaitForStateWithDedupeStatuses( + "LongRunningOrchestrator", runningInstanceId, expectedState: string.Empty, dedupeStatuses, expectedCode: HttpStatusCode.Conflict); + } + + [Fact] + [Trait("PowerShell", "Skip")] // Dedupe statuses not implemented in PowerShell + [Trait("Python", "Skip")] // Dedupe statuses not implemented in Python + [Trait("Node", "Skip")] // Dedupe statuses not implemented in Node + [Trait("Java", "Skip")] // Dedupe statuses not implemented in Java + public async Task StartOrchestrationWithInvalidDedupeStatusesFails() + { + // Dedupe statuses cannot have both "Terminated" and a running status (in this case "Pending") + List dedupeStatuses = ["Pending", "Failed", "Terminated"]; + + // We do not provide an expected state since we expect the request to fail + using HttpResponseMessage failedRequest = await StartAndWaitForStateWithDedupeStatuses( + "HelloCities", Guid.NewGuid().ToString(), expectedState: string.Empty, dedupeStatuses, expectedCode: HttpStatusCode.BadRequest); } private static async Task StartAndWaitForState( @@ -138,7 +181,8 @@ private static async Task StartAndWaitForStateWithDedupeSta DateTime? scheduledStartTime = null, HttpStatusCode expectedCode = HttpStatusCode.Accepted) { - string queryString = $"?orchestrationName={orchestrationName}&instanceId={instanceId}&dedupeStatuses={JsonSerializer.Serialize(dedupeStatuses)}"; + string queryString = $"?orchestrationName={orchestrationName}&instanceId={instanceId}" + + $"&dedupeStatuses={JsonSerializer.Serialize(dedupeStatuses)}"; if (scheduledStartTime is not null) { From dc292402253d1c1df2b72d0134b0329e99fcd635 Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Wed, 11 Feb 2026 12:46:09 -0800 Subject: [PATCH 20/43] added support to terminate existing running instances for restart --- .../ContextImplementations/DurableClient.cs | 10 - .../HttpApiHandler.cs | 4 + .../TaskHubGrpcServer.cs | 4 +- test/Common/DurableTaskEndToEndTests.cs | 342 +++++++++++------- test/Common/HttpApiHandlerTests.cs | 114 ++++-- test/e2e/Tests/Tests/DedupeStatusesTests.cs | 118 ++++-- .../Tests/Tests/RestartOrchestrationTests.cs | 29 +- 7 files changed, 426 insertions(+), 195 deletions(-) diff --git a/src/WebJobs.Extensions.DurableTask/ContextImplementations/DurableClient.cs b/src/WebJobs.Extensions.DurableTask/ContextImplementations/DurableClient.cs index a2fcc298c..9a16b6ce6 100644 --- a/src/WebJobs.Extensions.DurableTask/ContextImplementations/DurableClient.cs +++ b/src/WebJobs.Extensions.DurableTask/ContextImplementations/DurableClient.cs @@ -1157,16 +1157,6 @@ async Task IDurableOrchestrationClient.RestartAsync(string instanceId, b throw new ArgumentException($"An orchestrastion with the instanceId {instanceId} was not found."); } - bool isInstaceNotCompleted = status.RuntimeStatus == OrchestrationRuntimeStatus.Running || - status.RuntimeStatus == OrchestrationRuntimeStatus.Pending || - status.RuntimeStatus == OrchestrationRuntimeStatus.Suspended; - - if (isInstaceNotCompleted && !restartWithNewInstanceId) - { - throw new InvalidOperationException($"Instance '{instanceId}' cannot be restarted while it is in state '{status.RuntimeStatus}'. " + - "Wait until it has completed, or restart with a new instance ID."); - } - return restartWithNewInstanceId ? await ((IDurableOrchestrationClient)this).StartNewAsync(orchestratorFunctionName: status.Name, status.Input) : await ((IDurableOrchestrationClient)this).StartNewAsync(orchestratorFunctionName: status.Name, instanceId: status.InstanceId, status.Input); } diff --git a/src/WebJobs.Extensions.DurableTask/HttpApiHandler.cs b/src/WebJobs.Extensions.DurableTask/HttpApiHandler.cs index fba24b9a8..f4e680ccc 100644 --- a/src/WebJobs.Extensions.DurableTask/HttpApiHandler.cs +++ b/src/WebJobs.Extensions.DurableTask/HttpApiHandler.cs @@ -1034,6 +1034,10 @@ private async Task HandleRestartInstanceRequestAsync( { return request.CreateErrorResponse(HttpStatusCode.BadRequest, "InstanceId does not match a valid orchestration instance.", e); } + catch (OrchestrationAlreadyExistsException e) + { + return request.CreateErrorResponse(HttpStatusCode.BadRequest, "A non-terminal instance with this intance ID already exists.", e); + } catch (JsonReaderException e) { return request.CreateErrorResponse(HttpStatusCode.BadRequest, "Invalid JSON content", e); diff --git a/src/WebJobs.Extensions.DurableTask/TaskHubGrpcServer.cs b/src/WebJobs.Extensions.DurableTask/TaskHubGrpcServer.cs index 2e0541f5c..3ba86b862 100644 --- a/src/WebJobs.Extensions.DurableTask/TaskHubGrpcServer.cs +++ b/src/WebJobs.Extensions.DurableTask/TaskHubGrpcServer.cs @@ -494,9 +494,9 @@ await this.GetDurabilityProvider(context).CreateTaskOrchestrationAsync( // Thrown when th instanceId is not found. throw new RpcException(new Status(StatusCode.NotFound, $"ArgumentException: {ex.Message}")); } - catch (InvalidOperationException ex) + catch (OrchestrationAlreadyExistsException ex) { - throw new RpcException(new Status(StatusCode.FailedPrecondition, $"InvalidOperationException: {ex.Message}")); + throw new RpcException(new Status(StatusCode.FailedPrecondition, $"Non-terminal instance with this instance ID already exists: {ex.Message}")); } catch (Exception ex) { diff --git a/test/Common/DurableTaskEndToEndTests.cs b/test/Common/DurableTaskEndToEndTests.cs index 6dc7f8e2d..e96827a85 100644 --- a/test/Common/DurableTaskEndToEndTests.cs +++ b/test/Common/DurableTaskEndToEndTests.cs @@ -4746,6 +4746,38 @@ await Assert.ThrowsAsync(async () => } } + [Theory] + [Trait("Category", PlatformSpecificHelpers.TestCategory)] + [MemberData(nameof(GetBooleanAndFullFeaturedStorageProviderOptionsWithOverridableStatesAndSuspend))] + public async Task OverridableStates_RunningStatusesCorrectlyDeduped_ForRestart( + bool extendedSessions, + string storageProvider, + bool anyStateOverridable, + bool suspend) + { + await this.OverridableStates_RunningStatusesCorrectlyDeduped( + extendedSessions, + storageProvider, + anyStateOverridable, + suspend, + restart: true); + } + + [Theory] + [Trait("Category", PlatformSpecificHelpers.TestCategory)] + [MemberData(nameof(GetBooleanAndFullFeaturedStorageProviderOptionsWithOverridableStates))] + public async Task OverridableStates_TerminalStatusesAlwaysReusable_ForRestart( + bool extendedSessions, + string storageProvider, + bool anyStateOverridable) + { + await this.OverridableStates_TerminalStatusesAlwaysReusable( + extendedSessions, + storageProvider, + anyStateOverridable, + restart: true); + } + [Theory] [Trait("Category", PlatformSpecificHelpers.TestCategory)] [MemberData(nameof(TestDataGenerator.GetBooleanAndFullFeaturedStorageProviderOptions), MemberType = typeof(TestDataGenerator))] @@ -5304,136 +5336,30 @@ public static IEnumerable GetBooleanAndFullFeaturedStorageProviderOpti [Theory] [Trait("Category", PlatformSpecificHelpers.TestCategory)] [MemberData(nameof(GetBooleanAndFullFeaturedStorageProviderOptionsWithOverridableStatesAndSuspend))] - public async Task OverridableStates_RunningStatusesCorrectlyDeduped(bool extendedSessions, string storageProvider, bool anyStateOverridable, bool suspend) + public async Task OverridableStates_RunningStatusesCorrectlyDeduped_ForStartNew( + bool extendedSessions, + string storageProvider, + bool anyStateOverridable, + bool suspend) { - DurableTaskOptions options = new () - { - OverridableExistingInstanceStates = anyStateOverridable ? OverridableStates.AnyState : OverridableStates.NonRunningStates, - }; - - var instanceId = "OverridableStatesAnyStateTest_" + Guid.NewGuid().ToString("N"); - - using (ITestHost host = TestHelpers.GetJobHost( - this.loggerProvider, - nameof(this.OverridableStates_RunningStatusesCorrectlyDeduped), + await this.OverridableStates_RunningStatusesCorrectlyDeduped( extendedSessions, - storageProviderType: storageProvider, - options: options)) - { - await host.StartAsync(); - - int initialValue = 0; - - var client = await host.StartOrchestratorAsync(nameof(TestOrchestrations.Counter), initialValue, this.output, instanceId: instanceId); - - // Wait for the instance to go into the Running state. This is necessary to ensure log validation consistency. - await client.WaitForStartupAsync(this.output); - - TimeSpan waitTimeout = TimeSpan.FromSeconds(Debugger.IsAttached ? 300 : 10); - - // Perform some operations - await client.RaiseEventAsync("operation", "incr", this.output); - await client.WaitForCustomStatusAsync(waitTimeout, this.output, 1); - - // Make sure it's still running and didn't complete early (or fail). - var status = await client.GetStatusAsync(); - Assert.Equal(OrchestrationRuntimeStatus.Running, status?.RuntimeStatus); - - if (suspend) - { - await client.SuspendAsync("suspend for test"); - DurableOrchestrationStatus suspendedStatus = await client.WaitForStatusChange(this.output, OrchestrationRuntimeStatus.Suspended); - Assert.Equal(OrchestrationRuntimeStatus.Suspended, suspendedStatus?.RuntimeStatus); - } - - FunctionInvocationException invocationException = null; - try - { - await host.StartOrchestratorAsync(nameof(TestOrchestrations.Counter), initialValue, this.output, instanceId: instanceId); - } - catch (FunctionInvocationException caughtException) - { - invocationException = caughtException; - } - - await host.StopAsync(); - - // If any state is reusable, confirm that there is evidence the existing orchestration was terminated before the new one was created - if (anyStateOverridable && this.useTestLogger) - { - IReadOnlyCollection durableTaskCoreLogs = this.loggerProvider.CreatedLoggers.Single(l => l.Category == "DurableTask.Core").LogMessages; - Assert.Contains(durableTaskCoreLogs, log => log.ToString().StartsWith($"{instanceId}: Orchestration completed with a 'Terminated' status")); - } - - // Otherwise confirm that an exception was thrown when trying to create a new orchestration when one with a nonterminal status already exists - else if (!anyStateOverridable) - { - Assert.NotNull(invocationException); - Assert.NotNull(invocationException.InnerException); - Assert.IsType(invocationException.InnerException); - } - } + storageProvider, + anyStateOverridable, + suspend, + restart: false); } [Theory] [Trait("Category", PlatformSpecificHelpers.TestCategory)] [MemberData(nameof(GetBooleanAndFullFeaturedStorageProviderOptionsWithOverridableStates))] - public async Task OverridableStates_TerminalStatusesAlwaysReusable(bool extendedSessions, string storageProvider, bool anyStateOverridable) + public async Task OverridableStates_TerminalStatusesAlwaysReusable_ForStartNew(bool extendedSessions, string storageProvider, bool anyStateOverridable) { - DurableTaskOptions options = new () - { - OverridableExistingInstanceStates = anyStateOverridable ? OverridableStates.AnyState : OverridableStates.NonRunningStates, - }; - - string instanceIdBase = "OverridableStatesTerminalTest_" + Guid.NewGuid().ToString("N"); - - using ITestHost host = TestHelpers.GetJobHost( - this.loggerProvider, - nameof(this.OverridableStates_TerminalStatusesAlwaysReusable), + await this.OverridableStates_TerminalStatusesAlwaysReusable( extendedSessions, - storageProviderType: storageProvider, - options: options); - await host.StartAsync(); - - int initialValue = 0; - - // Test for all terminal statuses: Completed, Failed, Terminated - foreach (OrchestrationRuntimeStatus terminalStatus in new[] { OrchestrationRuntimeStatus.Completed, OrchestrationRuntimeStatus.Failed, OrchestrationRuntimeStatus.Terminated }) - { - string instanceId = instanceIdBase + "_" + terminalStatus; - - TestDurableClient client; - client = await host.StartOrchestratorAsync( - terminalStatus == OrchestrationRuntimeStatus.Failed ? nameof(TestOrchestrations.ThrowOrchestrator) : nameof(TestOrchestrations.Counter), - terminalStatus == OrchestrationRuntimeStatus.Failed ? string.Empty : initialValue, - this.output, - instanceId: instanceId); - - await client.WaitForStartupAsync(this.output); - DurableOrchestrationStatus status = null; - - if (terminalStatus == OrchestrationRuntimeStatus.Completed) - { - await client.RaiseEventAsync("operation", "end", this.output); - } - else if (terminalStatus == OrchestrationRuntimeStatus.Terminated) - { - await client.TerminateAsync("test terminate"); - } - - status = await client.WaitForCompletionAsync(this.output); - Assert.NotNull(status); - Assert.Equal(terminalStatus, status.RuntimeStatus); - - // Should always be able to start a new orchestration with the same instanceId - await host.StartOrchestratorAsync( - terminalStatus == OrchestrationRuntimeStatus.Failed ? nameof(TestOrchestrations.ThrowOrchestrator) : nameof(TestOrchestrations.Counter), - terminalStatus == OrchestrationRuntimeStatus.Failed ? string.Empty : initialValue, - this.output, - instanceId: instanceId); - } - - await host.StopAsync(); + storageProvider, + anyStateOverridable, + restart: false); } [Fact] @@ -6110,6 +6036,180 @@ private static void ValidateHttpManagementPayload(HttpManagementPayload httpMana httpManagementPayload.RestartPostUri); } + private async Task OverridableStates_RunningStatusesCorrectlyDeduped( + bool extendedSessions, + string storageProvider, + bool anyStateOverridable, + bool suspend, + bool restart) + { + DurableTaskOptions options = new () + { + OverridableExistingInstanceStates = anyStateOverridable ? OverridableStates.AnyState : OverridableStates.NonRunningStates, + }; + + string instanceId = Guid.NewGuid().ToString("N"); + + using ITestHost host = TestHelpers.GetJobHost( + this.loggerProvider, + restart ? nameof(this.OverridableStates_RunningStatusesCorrectlyDeduped_ForRestart) + : nameof(this.OverridableStates_RunningStatusesCorrectlyDeduped_ForStartNew), + extendedSessions, + storageProviderType: storageProvider, + options: options); + + await host.StartAsync(); + + int initialValue = 0; + + TestDurableClient client = await host.StartOrchestratorAsync( + nameof(TestOrchestrations.Counter), + initialValue, + this.output, + instanceId: instanceId); + + // Wait for the instance to go into the Running state. This is necessary to ensure log validation consistency. + await client.WaitForStartupAsync(this.output); + + var waitTimeout = TimeSpan.FromSeconds(Debugger.IsAttached ? 300 : 10); + + // Perform some operations + await client.RaiseEventAsync("operation", "incr", this.output); + await client.WaitForCustomStatusAsync(waitTimeout, this.output, 1); + + // Make sure it's still running and didn't complete early (or fail). + DurableOrchestrationStatus status = await client.GetStatusAsync(); + Assert.Equal(OrchestrationRuntimeStatus.Running, status?.RuntimeStatus); + + if (suspend) + { + await client.SuspendAsync("suspend for test"); + DurableOrchestrationStatus suspendedStatus = await client.WaitForStatusChange(this.output, OrchestrationRuntimeStatus.Suspended); + Assert.Equal(OrchestrationRuntimeStatus.Suspended, suspendedStatus?.RuntimeStatus); + } + + Exception exception = null; + try + { + if (restart) + { + await client.InnerClient.RestartAsync(instanceId, restartWithNewInstanceId: false); + } + else + { + await host.StartOrchestratorAsync(nameof(TestOrchestrations.Counter), initialValue, this.output, instanceId: instanceId); + } + } + catch (Exception caughtException) + { + exception = caughtException; + } + + await host.StopAsync(); + + // If any state is reusable, confirm that there is evidence the existing orchestration was terminated before the new one was created + if (anyStateOverridable && this.useTestLogger) + { + IReadOnlyCollection durableTaskCoreLogs = + this.loggerProvider.CreatedLoggers.Single(l => l.Category == "DurableTask.Core").LogMessages; + Assert.Contains(durableTaskCoreLogs, log => log.ToString().StartsWith($"{instanceId}: Orchestration completed with a 'Terminated' status")); + } + + // Otherwise confirm that an exception was thrown when trying to create a new orchestration when one with a nonterminal status already exists + else if (!anyStateOverridable) + { + Assert.NotNull(exception); + if (restart) + { + Assert.IsType(exception); + } + else + { + Assert.IsType(exception); + var functionInvocationException = (FunctionInvocationException)exception; + Assert.NotNull(functionInvocationException.InnerException); + Assert.IsType(functionInvocationException.InnerException); + } + } + } + + private async Task OverridableStates_TerminalStatusesAlwaysReusable( + bool extendedSessions, + string storageProvider, + bool anyStateOverridable, + bool restart) + { + DurableTaskOptions options = new () + { + OverridableExistingInstanceStates = anyStateOverridable ? OverridableStates.AnyState : OverridableStates.NonRunningStates, + }; + + string instanceIdBase = Guid.NewGuid().ToString("N"); + + using ITestHost host = TestHelpers.GetJobHost( + this.loggerProvider, + restart ? nameof(this.OverridableStates_TerminalStatusesAlwaysReusable_ForRestart) + : nameof(this.OverridableStates_TerminalStatusesAlwaysReusable_ForStartNew), + extendedSessions, + storageProviderType: storageProvider, + options: options); + await host.StartAsync(); + + int initialValue = 0; + + // Test for all terminal statuses: Completed, Failed, Terminated + foreach (OrchestrationRuntimeStatus terminalStatus in new[] + { + OrchestrationRuntimeStatus.Completed, + OrchestrationRuntimeStatus.Failed, + OrchestrationRuntimeStatus.Terminated, + }) + { + string instanceId = instanceIdBase + "_" + terminalStatus; + + TestDurableClient client; + client = await host.StartOrchestratorAsync( + terminalStatus == OrchestrationRuntimeStatus.Failed + ? nameof(TestOrchestrations.ThrowOrchestrator) : nameof(TestOrchestrations.Counter), + terminalStatus == OrchestrationRuntimeStatus.Failed ? string.Empty : initialValue, + this.output, + instanceId: instanceId); + + await client.WaitForStartupAsync(this.output); + DurableOrchestrationStatus status = null; + + if (terminalStatus == OrchestrationRuntimeStatus.Completed) + { + await client.RaiseEventAsync("operation", "end", this.output); + } + else if (terminalStatus == OrchestrationRuntimeStatus.Terminated) + { + await client.TerminateAsync("test terminate"); + } + + status = await client.WaitForCompletionAsync(this.output); + Assert.NotNull(status); + Assert.Equal(terminalStatus, status.RuntimeStatus); + + // Should always be able to start a new orchestration with the same instanceId + if (restart) + { + await client.InnerClient.RestartAsync(instanceId, restartWithNewInstanceId: false); + } + else + { + await host.StartOrchestratorAsync( + terminalStatus == OrchestrationRuntimeStatus.Failed + ? nameof(TestOrchestrations.ThrowOrchestrator) : nameof(TestOrchestrations.Counter), + terminalStatus == OrchestrationRuntimeStatus.Failed ? string.Empty : initialValue, + this.output, + instanceId: instanceId); + } + } + + await host.StopAsync(); + } + [DataContract] internal class ComplexType { diff --git a/test/Common/HttpApiHandlerTests.cs b/test/Common/HttpApiHandlerTests.cs index 70128ca02..7452fddf7 100644 --- a/test/Common/HttpApiHandlerTests.cs +++ b/test/Common/HttpApiHandlerTests.cs @@ -11,6 +11,7 @@ using System.Threading; using System.Threading.Tasks; using DurableTask.Core; +using DurableTask.Core.Exceptions; using DurableTask.Core.History; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; @@ -490,7 +491,8 @@ public async Task HandleGetStatusRequestAsync_Failed_Orchestration_Config_Respon { Method = HttpMethod.Get, RequestUri = getStatusRequestUriBuilder.Uri, - }); + }, + CancellationToken.None); Assert.Equal(statusCode, responseMessage.StatusCode); } @@ -532,7 +534,8 @@ public async Task GetAllStatus_is_Success() { Method = HttpMethod.Get, RequestUri = getStatusRequestUriBuilder.Uri, - }); + }, + CancellationToken.None); Assert.Equal(HttpStatusCode.OK, responseMessage.StatusCode); Assert.Equal(string.Empty, responseMessage.Headers.GetValues("x-ms-continuation-token").FirstOrDefault()); var actual = JsonConvert.DeserializeObject>(await responseMessage.Content.ReadAsStringAsync()); @@ -596,7 +599,8 @@ public async Task GetQueryStatus_is_Success() { Method = HttpMethod.Get, RequestUri = getStatusRequestUriBuilder.Uri, - }); + }, + CancellationToken.None); Assert.Equal(HttpStatusCode.OK, responseMessage.StatusCode); var actual = JsonConvert.DeserializeObject>(await responseMessage.Content.ReadAsStringAsync()); clientMock.Verify(x => x.ListInstancesAsync(It.IsAny(), It.IsAny())); @@ -672,7 +676,7 @@ public async Task GetQueryStatusWithPaging_is_Success() }; requestMessage.Headers.Add("x-ms-continuation-token", "XXXX-XXXXXXXX-XXXXXXXXXXXX"); - var responseMessage = await httpApiHandler.HandleRequestAsync(requestMessage); + var responseMessage = await httpApiHandler.HandleRequestAsync(requestMessage, CancellationToken.None); Assert.Equal(HttpStatusCode.OK, responseMessage.StatusCode); Assert.Equal("YYYY-YYYYYYYY-YYYYYYYYYYYY", responseMessage.Headers.GetValues("x-ms-continuation-token").FirstOrDefault()); var actual = JsonConvert.DeserializeObject>(await responseMessage.Content.ReadAsStringAsync()); @@ -738,7 +742,8 @@ public async Task GetQueryMultipleRuntimeStatus_is_Success() { Method = HttpMethod.Get, RequestUri = getStatusRequestUriBuilder.Uri, - }); + }, + CancellationToken.None); Assert.Equal(HttpStatusCode.OK, responseMessage.StatusCode); var actual = JsonConvert.DeserializeObject>(await responseMessage.Content.ReadAsStringAsync()); clientMock.Verify(x => x.ListInstancesAsync(It.IsAny(), It.IsAny())); @@ -796,7 +801,8 @@ public async Task GetQueryWithoutRuntimeStatus_is_Success() { Method = HttpMethod.Get, RequestUri = getStatusRequestUriBuilder.Uri, - }); + }, + CancellationToken.None); Assert.Equal(HttpStatusCode.OK, responseMessage.StatusCode); var actual = JsonConvert.DeserializeObject>(await responseMessage.Content.ReadAsStringAsync()); clientMock.Verify(x => x.ListInstancesAsync(It.IsAny(), It.IsAny())); @@ -837,7 +843,8 @@ public async Task HandleGetStatusRequestAsync_Correctly_Parses_InstanceId_With_S { Method = HttpMethod.Get, RequestUri = getStatusRequestUriBuilder.Uri, - }); + }, + CancellationToken.None); var actual = JsonConvert.DeserializeObject(await responseMessage.Content.ReadAsStringAsync()); Assert.Equal(HttpStatusCode.OK, responseMessage.StatusCode); @@ -883,7 +890,8 @@ await httpApiHandler.HandleRequestAsync( { Method = HttpMethod.Post, RequestUri = terminateRequestUriBuilder.Uri, - }); + }, + CancellationToken.None); Assert.Equal(testInstanceId, actualInstanceId); Assert.Equal(testReason, actualReason); @@ -928,7 +936,8 @@ await httpApiHandler.HandleRequestAsync( { Method = HttpMethod.Post, RequestUri = suspendRequestUriBuilder.Uri, - }); + }, + CancellationToken.None); Assert.Equal(testInstanceId, actualInstanceId); Assert.Equal(testReason, actualReason); @@ -973,7 +982,8 @@ await httpApiHandler.HandleRequestAsync( { Method = HttpMethod.Post, RequestUri = resumeRequestUriBuilder.Uri, - }); + }, + CancellationToken.None); Assert.Equal(testInstanceId, actualInstanceId); Assert.Equal(testReason, actualReason); @@ -1029,7 +1039,7 @@ public async Task RestartInstance_Is_Success(bool restartWithNewInstanceId) .Returns(testResponse); var httpApiHandler = new ExtendedHttpApiHandler(clientMock.Object); - var actualResponse = await httpApiHandler.HandleRequestAsync(testRequest); + var actualResponse = await httpApiHandler.HandleRequestAsync(testRequest, CancellationToken.None); Assert.Equal(HttpStatusCode.Accepted, actualResponse.StatusCode); var content = await actualResponse.Content.ReadAsStringAsync(); @@ -1094,7 +1104,7 @@ public async Task RestartInstanceAndWaitToComplete_Is_Success(bool restartWithNe .Returns(Task.FromResult(testResponse)); var httpApiHandler = new ExtendedHttpApiHandler(clientMock.Object); - var actualResponse = await httpApiHandler.HandleRequestAsync(testRequest); + var actualResponse = await httpApiHandler.HandleRequestAsync(testRequest, CancellationToken.None); Assert.Equal(HttpStatusCode.Accepted, actualResponse.StatusCode); var content = await actualResponse.Content.ReadAsStringAsync(); @@ -1130,7 +1140,7 @@ public async Task RestartInstance_Returns_HTTP_400_On_Invalid_InstanceId() .Throws(new ArgumentException()); var httpApiHandler = new ExtendedHttpApiHandler(clientMock.Object); - var actualResponse = await httpApiHandler.HandleRequestAsync(testRequest); + var actualResponse = await httpApiHandler.HandleRequestAsync(testRequest, CancellationToken.None); Assert.Equal(HttpStatusCode.BadRequest, actualResponse.StatusCode); var content = await actualResponse.Content.ReadAsStringAsync(); @@ -1138,6 +1148,35 @@ public async Task RestartInstance_Returns_HTTP_400_On_Invalid_InstanceId() Assert.Equal("InstanceId does not match a valid orchestration instance.", error["Message"].ToString()); } + [Fact] + [Trait("Category", PlatformSpecificHelpers.TestCategory)] + public async Task RestartInstance_Returns_HTTP_400_On_Invalid_Existing_Instance() + { + string testBadInstanceId = Guid.NewGuid().ToString("N"); + + var startRequestUriBuilder = new UriBuilder(TestConstants.NotificationUrl); + startRequestUriBuilder.Path += $"/Instances/{testBadInstanceId}/restart"; + + var testRequest = new HttpRequestMessage + { + Method = HttpMethod.Post, + RequestUri = startRequestUriBuilder.Uri, + }; + + var clientMock = new Mock(); + clientMock + .Setup(x => x.RestartAsync(It.IsAny(), It.IsAny())) + .Throws(new OrchestrationAlreadyExistsException()); + + var httpApiHandler = new ExtendedHttpApiHandler(clientMock.Object); + var actualResponse = await httpApiHandler.HandleRequestAsync(testRequest, CancellationToken.None); + + Assert.Equal(HttpStatusCode.BadRequest, actualResponse.StatusCode); + var content = await actualResponse.Content.ReadAsStringAsync(); + var error = JsonConvert.DeserializeObject(content); + Assert.Equal("A non-terminal instance with this intance ID already exists.", error["Message"].ToString()); + } + [Theory] [InlineData(null, false)] [InlineData(null, true)] @@ -1192,7 +1231,7 @@ public async Task StartNewInstance_Is_Success(string instanceId, bool hasContent .Returns(testResponse); var httpApiHandler = new ExtendedHttpApiHandler(clientMock.Object); - var actualResponse = await httpApiHandler.HandleRequestAsync(testRequest); + var actualResponse = await httpApiHandler.HandleRequestAsync(testRequest, CancellationToken.None); Assert.Equal(HttpStatusCode.Accepted, actualResponse.StatusCode); var content = await actualResponse.Content.ReadAsStringAsync(); @@ -1262,7 +1301,7 @@ public async Task StartNewInstanceAndWaitToComplete_Is_Success(string instanceId .Returns(Task.FromResult(testResponse)); var httpApiHandler = new ExtendedHttpApiHandler(clientMock.Object); - var actualResponse = await httpApiHandler.HandleRequestAsync(testRequest); + var actualResponse = await httpApiHandler.HandleRequestAsync(testRequest, CancellationToken.None); Assert.Equal(HttpStatusCode.Accepted, actualResponse.StatusCode); var content = await actualResponse.Content.ReadAsStringAsync(); @@ -1304,7 +1343,7 @@ public async Task StartNewInstance_Returns_HTTP_400_On_Bad_JSON() .Throws(new JsonReaderException()); var httpApiHandler = new ExtendedHttpApiHandler(clientMock.Object); - var actualResponse = await httpApiHandler.HandleRequestAsync(testRequest); + var actualResponse = await httpApiHandler.HandleRequestAsync(testRequest, CancellationToken.None); Assert.Equal(HttpStatusCode.BadRequest, actualResponse.StatusCode); var content = await actualResponse.Content.ReadAsStringAsync(); @@ -1336,7 +1375,7 @@ public async Task StartNewInstance_Returns_HTTP_400_On_Missing_Function() .Throws(new ArgumentException(exceptionMessage)); var httpApiHandler = new ExtendedHttpApiHandler(clientMock.Object); - var actualResponse = await httpApiHandler.HandleRequestAsync(testRequest); + var actualResponse = await httpApiHandler.HandleRequestAsync(testRequest, CancellationToken.None); Assert.Equal(HttpStatusCode.BadRequest, actualResponse.StatusCode); var content = await actualResponse.Content.ReadAsStringAsync(); @@ -1345,6 +1384,39 @@ public async Task StartNewInstance_Returns_HTTP_400_On_Missing_Function() Assert.Equal(exceptionMessage, error["ExceptionMessage"].ToString()); } + [Fact] + [Trait("Category", PlatformSpecificHelpers.TestCategory)] + public async Task StartNewInstance_Returns_HTTP_409_On_Existing_Orchestration() + { + string testInstanceId = Guid.NewGuid().ToString("N"); + string testFunctionName = "TestOrchestrator"; + + var startRequestUriBuilder = new UriBuilder(TestConstants.NotificationUrl); + startRequestUriBuilder.Path += $"/Orchestrators/{testFunctionName}"; + + var testRequest = new HttpRequestMessage + { + Method = HttpMethod.Post, + RequestUri = startRequestUriBuilder.Uri, + Content = new StringContent("\"TestContent\"", Encoding.UTF8, "application/json"), + }; + + string exceptionMessage = $"An orchestration with instance ID '{testInstanceId}' and status " + + $"'{OrchestrationStatus.Running}' already exists"; + var clientMock = new Mock(); + clientMock + .Setup(x => x.StartNewAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Throws(new OrchestrationAlreadyExistsException(exceptionMessage)); + + var httpApiHandler = new ExtendedHttpApiHandler(clientMock.Object); + var actualResponse = await httpApiHandler.HandleRequestAsync(testRequest, CancellationToken.None); + + Assert.Equal(HttpStatusCode.Conflict, actualResponse.StatusCode); + var content = await actualResponse.Content.ReadAsStringAsync(); + var error = JsonConvert.DeserializeObject(content); + Assert.Equal(exceptionMessage, error["Message"].ToString()); + } + [Theory] [InlineData(false, true)] [InlineData(false, false)] @@ -1379,7 +1451,7 @@ public async Task GetEntity_Returns_State_Or_HTTP_404(bool hasKey, bool exists) .Returns(Task.FromResult(result)); var httpApiHandler = new ExtendedHttpApiHandler(clientMock.Object); - var actualResponse = await httpApiHandler.HandleRequestAsync(testRequest); + var actualResponse = await httpApiHandler.HandleRequestAsync(testRequest, CancellationToken.None); if (exists) { @@ -1483,7 +1555,7 @@ public async Task Entities_Query_Calls_ListEntitiesAsync(bool useNameFilter, boo // Test HttpApiHandler response var httpApiHandler = new ExtendedHttpApiHandler(clientMock.Object); - HttpResponseMessage responseMessage = await httpApiHandler.HandleRequestAsync(requestMessage); + HttpResponseMessage responseMessage = await httpApiHandler.HandleRequestAsync(requestMessage, CancellationToken.None); Assert.Equal(HttpStatusCode.OK, responseMessage.StatusCode); clientMock.Verify(x => x.ListEntitiesAsync(It.IsAny(), It.IsAny())); @@ -1588,7 +1660,7 @@ public async Task SignalEntity_Is_Success(bool hasKey, bool hasOp, bool hasConte } var httpApiHandler = new ExtendedHttpApiHandler(clientMock.Object); - var actualResponse = await httpApiHandler.HandleRequestAsync(testRequest); + var actualResponse = await httpApiHandler.HandleRequestAsync(testRequest, CancellationToken.None); Assert.Equal(HttpStatusCode.Accepted, actualResponse.StatusCode); } @@ -1723,7 +1795,7 @@ public async Task StartNewInstance_Calls_CreateTaskOrchestrationAsync_With_Corre var handler = new HttpApiHandler(customExtension, NullLogger.Instance); var request = new HttpRequestMessage(HttpMethod.Post, requestUri); - var response = await handler.HandleRequestAsync(request); + var response = await handler.HandleRequestAsync(request, CancellationToken.None); // Verify mock interactions orchestrationServiceClientMock.Verify( diff --git a/test/e2e/Tests/Tests/DedupeStatusesTests.cs b/test/e2e/Tests/Tests/DedupeStatusesTests.cs index d1699fdb9..9cccc2052 100644 --- a/test/e2e/Tests/Tests/DedupeStatusesTests.cs +++ b/test/e2e/Tests/Tests/DedupeStatusesTests.cs @@ -1,5 +1,7 @@ using System.Net; using System.Text.Json; +using Microsoft.Azure.WebJobs.Extensions.DurableTask; +using Microsoft.DurableTask.Protobuf; using Xunit; using Xunit.Abstractions; @@ -19,8 +21,10 @@ public DedupeStatusesTests(FunctionAppFixture fixture, ITestOutputHelper testOut } [Fact] - public async Task CanStartOrchestrationWithSameIdForAllStatusesForEmptyDedupeStatuses() + public async Task CanStartOrchestration_WithSameId_ForAllStatuses_ForEmptyDedupeStatuses() { + HttpResponseMessage terminateResponse; + // Completed string completedInstanceId = Guid.NewGuid().ToString(); using HttpResponseMessage startCompletedResponseFirstAttempt = await StartAndWaitForState( @@ -47,6 +51,10 @@ public async Task CanStartOrchestrationWithSameIdForAllStatusesForEmptyDedupeSta await TerminateAndWaitForState(terminatedInstanceId, startTerminatedResponseFirstAttempt); using HttpResponseMessage startTerminatedResponseSecondAttempt = await StartAndWaitForState( "LongRunningOrchestrator", terminatedInstanceId, "Running"); + // Clean-up + terminateResponse = await HttpHelpers.InvokeHttpTrigger("TerminateInstance", $"?instanceId={terminatedInstanceId}"); + Assert.Equal(HttpStatusCode.OK, terminateResponse.StatusCode); + terminateResponse.Dispose(); } // Pending @@ -69,6 +77,10 @@ public async Task CanStartOrchestrationWithSameIdForAllStatusesForEmptyDedupeSta "LongRunningOrchestrator", runningInstanceId, "Running"); using HttpResponseMessage startRunningResponseSecondAttempt = await StartAndWaitForState( "LongRunningOrchestrator", runningInstanceId, "Running"); + // Clean-up + terminateResponse = await HttpHelpers.InvokeHttpTrigger("TerminateInstance", $"?instanceId={runningInstanceId}"); + Assert.Equal(HttpStatusCode.OK, terminateResponse.StatusCode); + terminateResponse.Dispose(); // Suspended string suspendedInstanceId = Guid.NewGuid().ToString(); @@ -77,23 +89,33 @@ public async Task CanStartOrchestrationWithSameIdForAllStatusesForEmptyDedupeSta await SuspendAndWaitForState(suspendedInstanceId, startSuspendedResponseFirstAttempt); using HttpResponseMessage startSuspendedResponseSecondAttempt = await StartAndWaitForState( "LongRunningOrchestrator", suspendedInstanceId, "Running"); + // Clean-up + terminateResponse = await HttpHelpers.InvokeHttpTrigger("TerminateInstance", $"?instanceId={suspendedInstanceId}"); + Assert.Equal(HttpStatusCode.OK, terminateResponse.StatusCode); + terminateResponse.Dispose(); } - [Fact] + [Theory] [Trait("PowerShell", "Skip")] // Dedupe statuses not implemented in PowerShell [Trait("Python", "Skip")] // Dedupe statuses not implemented in Python [Trait("Node", "Skip")] // Dedupe statuses not implemented in Node [Trait("Java", "Skip")] // Dedupe statuses not implemented in Java - public async Task StartOrchestrationWithSameIdFailsForDedupeStatuses() + [InlineData([])] + [InlineData("Pending", "Failed")] + public async Task StartOrchestration_WithSameId_FailsIfExistingStatus_InDedupeStatuses(params string[] dedupeStatuses) { - List dedupeStatuses = ["Running", "Failed"]; + HttpResponseMessage terminateResponse; - // Completed, should succeed + // Completed string completedInstanceId = Guid.NewGuid().ToString(); using HttpResponseMessage startCompletedResponseFirstAttempt = await StartAndWaitForStateWithDedupeStatuses( "HelloCities", completedInstanceId, "Completed", dedupeStatuses); using HttpResponseMessage startCompletedResponseSecondAttempt = await StartAndWaitForStateWithDedupeStatuses( - "HelloCities", completedInstanceId, "Completed", dedupeStatuses); + "HelloCities", + completedInstanceId, + "Completed", + dedupeStatuses, + expectedCode: dedupeStatuses.Contains("Completed") ? HttpStatusCode.Conflict : HttpStatusCode.Accepted); // Terminated string terminatedInstanceId = Guid.NewGuid().ToString(); @@ -101,7 +123,29 @@ public async Task StartOrchestrationWithSameIdFailsForDedupeStatuses() "LongRunningOrchestrator", terminatedInstanceId, "Running", dedupeStatuses); await TerminateAndWaitForState(terminatedInstanceId, startTerminatedResponseFirstAttempt); using HttpResponseMessage startTerminatedResponseSecondAttempt = await StartAndWaitForStateWithDedupeStatuses( - "LongRunningOrchestrator", terminatedInstanceId, "Running", dedupeStatuses); + "LongRunningOrchestrator", + terminatedInstanceId, + "Running", + dedupeStatuses, + expectedCode: dedupeStatuses.Contains("Terminated") ? HttpStatusCode.Conflict : HttpStatusCode.Accepted); + // Clean-up + if (!dedupeStatuses.Contains("Terminated")) + { + terminateResponse = await HttpHelpers.InvokeHttpTrigger("TerminateInstance", $"?instanceId={terminatedInstanceId}"); + Assert.Equal(HttpStatusCode.OK, terminateResponse.StatusCode); + terminateResponse.Dispose(); + } + + // Failed + string failedInstanceId = Guid.NewGuid().ToString(); + using HttpResponseMessage startFailedResponseFirstAttempt = await StartAndWaitForStateWithDedupeStatuses( + "RethrowActivityException", failedInstanceId, "Failed", dedupeStatuses); + using HttpResponseMessage startFailedResponseSecondAttempt = await StartAndWaitForStateWithDedupeStatuses( + "RethrowActivityException", + failedInstanceId, + "Completed", + dedupeStatuses, + expectedCode: dedupeStatuses.Contains("Failed") ? HttpStatusCode.Conflict : HttpStatusCode.Accepted); // Pending string pendingInstanceId = Guid.NewGuid().ToString(); @@ -109,7 +153,26 @@ public async Task StartOrchestrationWithSameIdFailsForDedupeStatuses() using HttpResponseMessage startPendingResponseFirstAttempt = await StartAndWaitForStateWithDedupeStatuses( "HelloCities", pendingInstanceId, "Pending", dedupeStatuses, scheduledStartTime: scheduledStartTime); using HttpResponseMessage startPendingResponseSecondAttempt = await StartAndWaitForStateWithDedupeStatuses( - "HelloCities", pendingInstanceId, "Completed", dedupeStatuses); + "HelloCities", + pendingInstanceId, + "Completed", + dedupeStatuses, + expectedCode: dedupeStatuses.Contains("Pending") ? HttpStatusCode.Conflict : HttpStatusCode.Accepted); + + // Running + string runningInstanceId = Guid.NewGuid().ToString(); + using HttpResponseMessage startRunningResponseFirstAttempt = await StartAndWaitForStateWithDedupeStatuses( + "LongRunningOrchestrator", runningInstanceId, "Running", dedupeStatuses); + using HttpResponseMessage startRunningResponseSecondAttempt = await StartAndWaitForStateWithDedupeStatuses( + "LongRunningOrchestrator", + runningInstanceId, + expectedState: "Running", + dedupeStatuses, + expectedCode: dedupeStatuses.Contains("Running") ? HttpStatusCode.Conflict : HttpStatusCode.Accepted); + // Clean-up + terminateResponse = await HttpHelpers.InvokeHttpTrigger("TerminateInstance", $"?instanceId={runningInstanceId}"); + Assert.Equal(HttpStatusCode.OK, terminateResponse.StatusCode); + terminateResponse.Dispose(); // Suspended string suspendedInstanceId = Guid.NewGuid().ToString(); @@ -117,35 +180,28 @@ public async Task StartOrchestrationWithSameIdFailsForDedupeStatuses() "LongRunningOrchestrator", suspendedInstanceId, "Running", dedupeStatuses); await SuspendAndWaitForState(suspendedInstanceId, startSuspendedResponseFirstAttempt); using HttpResponseMessage startSuspendedResponseSecondAttempt = await StartAndWaitForStateWithDedupeStatuses( - "LongRunningOrchestrator", suspendedInstanceId, "Running", dedupeStatuses); - - // Failed, should fail - string failedInstanceId = Guid.NewGuid().ToString(); - using HttpResponseMessage startFailedResponseFirstAttempt = await StartAndWaitForStateWithDedupeStatuses( - "RethrowActivityException", failedInstanceId, "Failed", dedupeStatuses); - // We do not provide an expected state since we expect the request to fail - using HttpResponseMessage startFailedResponseSecondAttempt = await StartAndWaitForStateWithDedupeStatuses( - "RethrowActivityException", failedInstanceId, expectedState: string.Empty, dedupeStatuses, expectedCode: HttpStatusCode.Conflict); - - // Running, should fail - string runningInstanceId = Guid.NewGuid().ToString(); - using HttpResponseMessage startRunningResponseFirstAttempt = await StartAndWaitForStateWithDedupeStatuses( - "LongRunningOrchestrator", runningInstanceId, "Running", dedupeStatuses); - // We do not provide an expected state since we expect the request to fail - using HttpResponseMessage startRunningResponseSecondAttempt = await StartAndWaitForStateWithDedupeStatuses( - "LongRunningOrchestrator", runningInstanceId, expectedState: string.Empty, dedupeStatuses, expectedCode: HttpStatusCode.Conflict); + "LongRunningOrchestrator", + suspendedInstanceId, + "Running", + dedupeStatuses, + expectedCode: dedupeStatuses.Contains("Suspended") ? HttpStatusCode.Conflict : HttpStatusCode.Accepted); + // Clean-up + terminateResponse = await HttpHelpers.InvokeHttpTrigger("TerminateInstance", $"?instanceId={suspendedInstanceId}"); + Assert.Equal(HttpStatusCode.OK, terminateResponse.StatusCode); + terminateResponse.Dispose(); } - [Fact] + [Theory] [Trait("PowerShell", "Skip")] // Dedupe statuses not implemented in PowerShell [Trait("Python", "Skip")] // Dedupe statuses not implemented in Python [Trait("Node", "Skip")] // Dedupe statuses not implemented in Node [Trait("Java", "Skip")] // Dedupe statuses not implemented in Java - public async Task StartOrchestrationWithInvalidDedupeStatusesFails() + [InlineData("Pending", "Failed", "Terminated")] + [InlineData("Running", "Failed", "Terminated")] + [InlineData("Suspended", "Failed", "Terminated")] + public async Task StartOrchestration_WithInvalidDedupeStatuses_ThrowsArgumentException(params string[] dedupeStatuses) { - // Dedupe statuses cannot have both "Terminated" and a running status (in this case "Pending") - List dedupeStatuses = ["Pending", "Failed", "Terminated"]; - + // Dedupe statuses cannot have both "Terminated" and a running status // We do not provide an expected state since we expect the request to fail using HttpResponseMessage failedRequest = await StartAndWaitForStateWithDedupeStatuses( "HelloCities", Guid.NewGuid().ToString(), expectedState: string.Empty, dedupeStatuses, expectedCode: HttpStatusCode.BadRequest); @@ -177,7 +233,7 @@ private static async Task StartAndWaitForStateWithDedupeSta string orchestrationName, string instanceId, string expectedState, - List dedupeStatuses, + string[] dedupeStatuses, DateTime? scheduledStartTime = null, HttpStatusCode expectedCode = HttpStatusCode.Accepted) { diff --git a/test/e2e/Tests/Tests/RestartOrchestrationTests.cs b/test/e2e/Tests/Tests/RestartOrchestrationTests.cs index fe627a7c2..0799bc469 100644 --- a/test/e2e/Tests/Tests/RestartOrchestrationTests.cs +++ b/test/e2e/Tests/Tests/RestartOrchestrationTests.cs @@ -3,6 +3,7 @@ using System.Net; using System.Text.Json; +using Microsoft.VisualStudio.TestPlatform.Utilities; using Xunit; using Xunit.Abstractions; @@ -115,15 +116,16 @@ public async Task RestartOrchestration_NonExistentInstanceId_ShouldReturnNotFoun [Trait("Java", "Skip")] // RestartAsync not yet implemented in Java [Trait("Python", "Skip")] // RestartAsync not supported in Python [Trait("Node", "Skip")] // RestartAsync not supported in Node - // Test that if we restart a instance that doesn't reach to completed state, - // If RestartWithNewInstanceId is set to false, a InvalidOperationException error will be thrown. - public async Task RestartOrchestration_NotCompletedOrchestrationWithRestartFalse_ShouldReturnFailedPrecondition() + public async Task RestartOrchestration_NotCompletedOrchestrationWithRestartFalse_ShouldSucceed() { // Start a long-running orchestration using HttpResponseMessage response = await HttpHelpers.InvokeHttpTrigger("RestarttOrchestration_HttpStart/LongOrchestrator"); Assert.Equal(HttpStatusCode.Accepted, response.StatusCode); string instanceId = await DurableHelpers.ParseInstanceIdAsync(response); string statusQueryGetUri = await DurableHelpers.ParseStatusQueryGetUriAsync(response); + DurableHelpers.OrchestrationStatusDetails orchestrationDetails + = await DurableHelpers.GetRunningOrchestrationDetailsAsync(statusQueryGetUri); + DateTime createdTime1 = orchestrationDetails.CreatedTime; // Wait for the orchestration to be running await DurableHelpers.WaitForOrchestrationStateAsync(statusQueryGetUri, "Running", 30); @@ -137,15 +139,22 @@ public async Task RestartOrchestration_NotCompletedOrchestrationWithRestartFalse string jsonBody = JsonSerializer.Serialize(restartPayload); + // Restart the orchestrator with the same instance id) using HttpResponseMessage restartResponse = await HttpHelpers.InvokeHttpTriggerWithBody( - "RestartOrchestration_HttpRestartWithErrorHandling", jsonBody, "application/json"); + "RestartOrchestration_HttpRestart", jsonBody, "application/json"); + Assert.Equal(HttpStatusCode.Accepted, restartResponse.StatusCode); - Assert.Equal(HttpStatusCode.BadRequest, restartResponse.StatusCode); - - string responseContent = await restartResponse.Content.ReadAsStringAsync(); - - // Verify the returned exception contains the correct information. - Assert.Contains(fixture.functionLanguageLocalizer.GetLocalizedStringValue("RestartRunningInstance.ErrorMessage", instanceId), responseContent); + string restartStatusQueryGetUri = await DurableHelpers.ParseStatusQueryGetUriAsync(restartResponse); + string restartInstanceId = await DurableHelpers.ParseInstanceIdAsync(restartResponse); + + await DurableHelpers.WaitForOrchestrationStateAsync(restartStatusQueryGetUri, "Running", 30); + DurableHelpers.OrchestrationStatusDetails restartOrchestrationDetails + = await DurableHelpers.GetRunningOrchestrationDetailsAsync(restartStatusQueryGetUri); + DateTime createdTime2 = restartOrchestrationDetails.CreatedTime; + + // Created time should be different. + Assert.NotEqual(createdTime1, createdTime2); + Assert.Equal(instanceId, restartInstanceId); // Clean up: terminate the long-running orchestration using HttpResponseMessage terminateResponse = await HttpHelpers.InvokeHttpTrigger("TerminateInstance", $"?instanceId={instanceId}"); From 7e53ac6f6660067cdb490a8538785071028b95dd Mon Sep 17 00:00:00 2001 From: sophiatev <38052607+sophiatev@users.noreply.github.com> Date: Wed, 11 Feb 2026 13:03:26 -0800 Subject: [PATCH 21/43] Update src/WebJobs.Extensions.DurableTask/HttpApiHandler.cs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/WebJobs.Extensions.DurableTask/HttpApiHandler.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/WebJobs.Extensions.DurableTask/HttpApiHandler.cs b/src/WebJobs.Extensions.DurableTask/HttpApiHandler.cs index 39785b9f6..803d044b3 100644 --- a/src/WebJobs.Extensions.DurableTask/HttpApiHandler.cs +++ b/src/WebJobs.Extensions.DurableTask/HttpApiHandler.cs @@ -14,7 +14,6 @@ using DurableTask.Core; using DurableTask.Core.Exceptions; using DurableTask.Core.History; -using Grpc.Core; using Microsoft.AspNetCore.Routing; using Microsoft.AspNetCore.Routing.Template; using Microsoft.Azure.WebJobs.Extensions.DurableTask.Correlation; From ac48ed70209afb555b4279b619475bbc0f622173 Mon Sep 17 00:00:00 2001 From: sophiatev <38052607+sophiatev@users.noreply.github.com> Date: Wed, 11 Feb 2026 13:03:47 -0800 Subject: [PATCH 22/43] Update test/e2e/Tests/Tests/DedupeStatusesTests.cs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- test/e2e/Tests/Tests/DedupeStatusesTests.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/test/e2e/Tests/Tests/DedupeStatusesTests.cs b/test/e2e/Tests/Tests/DedupeStatusesTests.cs index 9cccc2052..ccff86dca 100644 --- a/test/e2e/Tests/Tests/DedupeStatusesTests.cs +++ b/test/e2e/Tests/Tests/DedupeStatusesTests.cs @@ -1,7 +1,6 @@ using System.Net; using System.Text.Json; using Microsoft.Azure.WebJobs.Extensions.DurableTask; -using Microsoft.DurableTask.Protobuf; using Xunit; using Xunit.Abstractions; From 25d89320af8304908a2097b70100c5272a2f2e01 Mon Sep 17 00:00:00 2001 From: sophiatev <38052607+sophiatev@users.noreply.github.com> Date: Wed, 11 Feb 2026 13:04:03 -0800 Subject: [PATCH 23/43] Update test/e2e/Tests/Tests/RestartOrchestrationTests.cs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .../Tests/Tests/RestartOrchestrationTests.cs | 343 +++++++++--------- 1 file changed, 171 insertions(+), 172 deletions(-) diff --git a/test/e2e/Tests/Tests/RestartOrchestrationTests.cs b/test/e2e/Tests/Tests/RestartOrchestrationTests.cs index fdef1015f..5b4ed09c0 100644 --- a/test/e2e/Tests/Tests/RestartOrchestrationTests.cs +++ b/test/e2e/Tests/Tests/RestartOrchestrationTests.cs @@ -1,173 +1,172 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the MIT License. See License.txt in the project root for license information. - -using System.Net; -using System.Text.Json; -using Microsoft.VisualStudio.TestPlatform.Utilities; -using Xunit; -using Xunit.Abstractions; - -namespace Microsoft.Azure.Durable.Tests.DotnetIsolatedE2E; - -[Collection(Constants.FunctionAppCollectionName)] -public class RestartOrchestrationTests -{ - private readonly FunctionAppFixture fixture; - private readonly ITestOutputHelper output; - - public RestartOrchestrationTests(FunctionAppFixture fixture, ITestOutputHelper testOutputHelper) - { - this.fixture = fixture; - this.fixture.TestLogs.UseTestLogger(testOutputHelper); - this.output = testOutputHelper; - } - - [Theory] - [InlineData(false)] - [InlineData(true)] - [Trait("PowerShell", "Skip")] // RestartAsync not yet implemented in PowerShell - [Trait("Java", "Skip")] // RestartAsync not yet implemented in Java - [Trait("Python", "Skip")] // RestartAsync not supported in Python - [Trait("Node", "Skip")] // RestartAsync not supported in Node - [Trait("MSSQL", "Skip")] // MSSQL doesn't support tags at ExecutionStarted Event. - // Test behavior of restartasync of durabletaskclient. - // When restart with a instanceid and startwithnewinstanceid is false, the orchestration should be restarted with the same instance id. - // When restart with a instanceid and startwithnewinstanceid is true, the orchestration should be restarted with a new instance id. - public async Task RestartOrchestration_CreatedTimeAndOutputChange(bool restartWithNewInstanceId) - { - // Start the orchestration - using HttpResponseMessage response = await HttpHelpers.InvokeHttpTrigger("RestarttOrchestration_HttpStart/SimpleOrchestrator"); - Assert.Equal(HttpStatusCode.Accepted, response.StatusCode); - string statusQueryGetUri = await DurableHelpers.ParseStatusQueryGetUriAsync(response); - string instanceId = await DurableHelpers.ParseInstanceIdAsync(response); - - await DurableHelpers.WaitForOrchestrationStateAsync(statusQueryGetUri, "Completed", 30); - var orchestrationDetails = await DurableHelpers.GetRunningOrchestrationDetailsAsync(statusQueryGetUri); - string output1 = orchestrationDetails.Output; - DateTime createdTime1 = orchestrationDetails.CreatedTime; - - // best practice to wait for 1 seconds before restarting orchestration to avoid race condition. - await Task.Delay(1000); - - var restartPayload = new { - InstanceId = instanceId, - RestartWithNewInstanceId = restartWithNewInstanceId - }; - - string jsonBody = JsonSerializer.Serialize(restartPayload); - - // Restart the orchestrator with the same instance id) - using HttpResponseMessage restartResponse = await HttpHelpers.InvokeHttpTriggerWithBody( - "RestartOrchestration_HttpRestart", jsonBody, "application/json"); - Assert.Equal(HttpStatusCode.Accepted, restartResponse.StatusCode); - string restartStatusQueryGetUri = await DurableHelpers.ParseStatusQueryGetUriAsync(restartResponse); - string restartInstanceId = await DurableHelpers.ParseInstanceIdAsync(restartResponse); - - await DurableHelpers.WaitForOrchestrationStateAsync(restartStatusQueryGetUri, "Completed", 30); - var restartOrchestrationDetails = await DurableHelpers.GetRunningOrchestrationDetailsAsync(restartStatusQueryGetUri); - string output2 = restartOrchestrationDetails.Output; - DateTime createdTime2 = restartOrchestrationDetails.CreatedTime; - - // The outputs should be the same as input is same. - Assert.Equal(output1, output2); - // Created time should be different. - Assert.NotEqual(createdTime1, createdTime2); - - if (restartWithNewInstanceId) - { - // If restartWithNewInstanceId is True, the two instanceId should be different. - Assert.NotEqual(instanceId, restartInstanceId); - } - else - { - Assert.Equal(instanceId, restartInstanceId); - } - - HttpResponseMessage result = await HttpHelpers.InvokeHttpTrigger("RestartOrchestrator_Query_Tags", $"?id={restartInstanceId}"); - - // Verify that restarted orchestration instance contains tags. - Assert.Equal(HttpStatusCode.OK, result.StatusCode); - var content = await result.Content.ReadAsStringAsync(); - // Content: { "output": "...", "tags": { "testtag": "true" } } - Assert.Contains("output", content); - Assert.Contains("testtag", content); - } - - [Fact] - [Trait("PowerShell", "Skip")] // RestartAsync not yet implemented in PowerShell - [Trait("Java", "Skip")] // RestartAsync not yet implemented in Java - [Trait("Python", "Skip")] // RestartAsync not supported in Python - [Trait("Node", "Skip")] // RestartAsync not supported in Node - // Test that if we restart a instanceId that doesn't exist. We will throw ArgumentException exception. - public async Task RestartOrchestration_NonExistentInstanceId_ShouldReturnNotFound() - { - const string testInstanceId = "nonexistid"; - - // Test restarting with a non-existent instance ID - var restartPayload = new - { - InstanceId = testInstanceId, - RestartWithNewInstanceId = false - }; - - string jsonBody = JsonSerializer.Serialize(restartPayload); - - using HttpResponseMessage restartResponse = await HttpHelpers.InvokeHttpTriggerWithBody( - "RestartOrchestration_HttpRestartWithErrorHandling", jsonBody, "application/json"); - - string responseContent = await restartResponse.Content.ReadAsStringAsync(); - - // Verfity we weill return the right exception message. - Assert.Contains(fixture.functionLanguageLocalizer.GetLocalizedStringValue("RestartInvalidInstance.ErrorMessage", testInstanceId), responseContent); - } - - [Fact] - [Trait("PowerShell", "Skip")] // RestartAsync not yet implemented in PowerShell - [Trait("Java", "Skip")] // RestartAsync not yet implemented in Java - [Trait("Python", "Skip")] // RestartAsync not supported in Python - [Trait("Node", "Skip")] // RestartAsync not supported in Node - public async Task RestartOrchestration_NotCompletedOrchestrationWithRestartFalse_ShouldSucceed() - { - // Start a long-running orchestration - using HttpResponseMessage response = await HttpHelpers.InvokeHttpTrigger("RestarttOrchestration_HttpStart/LongOrchestrator"); - Assert.Equal(HttpStatusCode.Accepted, response.StatusCode); - string instanceId = await DurableHelpers.ParseInstanceIdAsync(response); - string statusQueryGetUri = await DurableHelpers.ParseStatusQueryGetUriAsync(response); - DurableHelpers.OrchestrationStatusDetails orchestrationDetails - = await DurableHelpers.GetRunningOrchestrationDetailsAsync(statusQueryGetUri); - DateTime createdTime1 = orchestrationDetails.CreatedTime; - - // Wait for the orchestration to be running - await DurableHelpers.WaitForOrchestrationStateAsync(statusQueryGetUri, "Running", 30); - - // Try to restart the running orchestration with restartWithNewInstanceId = false - var restartPayload = new - { - InstanceId = instanceId, - RestartWithNewInstanceId = false - }; - - string jsonBody = JsonSerializer.Serialize(restartPayload); - - // Restart the orchestrator with the same instance id) - using HttpResponseMessage restartResponse = await HttpHelpers.InvokeHttpTriggerWithBody( - "RestartOrchestration_HttpRestart", jsonBody, "application/json"); - Assert.Equal(HttpStatusCode.Accepted, restartResponse.StatusCode); - - string restartStatusQueryGetUri = await DurableHelpers.ParseStatusQueryGetUriAsync(restartResponse); - string restartInstanceId = await DurableHelpers.ParseInstanceIdAsync(restartResponse); - - await DurableHelpers.WaitForOrchestrationStateAsync(restartStatusQueryGetUri, "Running", 30); - DurableHelpers.OrchestrationStatusDetails restartOrchestrationDetails - = await DurableHelpers.GetRunningOrchestrationDetailsAsync(restartStatusQueryGetUri); - DateTime createdTime2 = restartOrchestrationDetails.CreatedTime; - - // Created time should be different. +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System.Net; +using System.Text.Json; +using Xunit; +using Xunit.Abstractions; + +namespace Microsoft.Azure.Durable.Tests.DotnetIsolatedE2E; + +[Collection(Constants.FunctionAppCollectionName)] +public class RestartOrchestrationTests +{ + private readonly FunctionAppFixture fixture; + private readonly ITestOutputHelper output; + + public RestartOrchestrationTests(FunctionAppFixture fixture, ITestOutputHelper testOutputHelper) + { + this.fixture = fixture; + this.fixture.TestLogs.UseTestLogger(testOutputHelper); + this.output = testOutputHelper; + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + [Trait("PowerShell", "Skip")] // RestartAsync not yet implemented in PowerShell + [Trait("Java", "Skip")] // RestartAsync not yet implemented in Java + [Trait("Python", "Skip")] // RestartAsync not supported in Python + [Trait("Node", "Skip")] // RestartAsync not supported in Node + [Trait("MSSQL", "Skip")] // MSSQL doesn't support tags at ExecutionStarted Event. + // Test behavior of restartasync of durabletaskclient. + // When restart with a instanceid and startwithnewinstanceid is false, the orchestration should be restarted with the same instance id. + // When restart with a instanceid and startwithnewinstanceid is true, the orchestration should be restarted with a new instance id. + public async Task RestartOrchestration_CreatedTimeAndOutputChange(bool restartWithNewInstanceId) + { + // Start the orchestration + using HttpResponseMessage response = await HttpHelpers.InvokeHttpTrigger("RestarttOrchestration_HttpStart/SimpleOrchestrator"); + Assert.Equal(HttpStatusCode.Accepted, response.StatusCode); + string statusQueryGetUri = await DurableHelpers.ParseStatusQueryGetUriAsync(response); + string instanceId = await DurableHelpers.ParseInstanceIdAsync(response); + + await DurableHelpers.WaitForOrchestrationStateAsync(statusQueryGetUri, "Completed", 30); + var orchestrationDetails = await DurableHelpers.GetRunningOrchestrationDetailsAsync(statusQueryGetUri); + string output1 = orchestrationDetails.Output; + DateTime createdTime1 = orchestrationDetails.CreatedTime; + + // best practice to wait for 1 seconds before restarting orchestration to avoid race condition. + await Task.Delay(1000); + + var restartPayload = new { + InstanceId = instanceId, + RestartWithNewInstanceId = restartWithNewInstanceId + }; + + string jsonBody = JsonSerializer.Serialize(restartPayload); + + // Restart the orchestrator with the same instance id) + using HttpResponseMessage restartResponse = await HttpHelpers.InvokeHttpTriggerWithBody( + "RestartOrchestration_HttpRestart", jsonBody, "application/json"); + Assert.Equal(HttpStatusCode.Accepted, restartResponse.StatusCode); + string restartStatusQueryGetUri = await DurableHelpers.ParseStatusQueryGetUriAsync(restartResponse); + string restartInstanceId = await DurableHelpers.ParseInstanceIdAsync(restartResponse); + + await DurableHelpers.WaitForOrchestrationStateAsync(restartStatusQueryGetUri, "Completed", 30); + var restartOrchestrationDetails = await DurableHelpers.GetRunningOrchestrationDetailsAsync(restartStatusQueryGetUri); + string output2 = restartOrchestrationDetails.Output; + DateTime createdTime2 = restartOrchestrationDetails.CreatedTime; + + // The outputs should be the same as input is same. + Assert.Equal(output1, output2); + // Created time should be different. Assert.NotEqual(createdTime1, createdTime2); - Assert.Equal(instanceId, restartInstanceId); - - // Clean up: terminate the long-running orchestration - using HttpResponseMessage terminateResponse = await HttpHelpers.InvokeHttpTrigger("TerminateInstance", $"?instanceId={instanceId}"); - Assert.Equal(HttpStatusCode.OK, terminateResponse.StatusCode); - } -} + + if (restartWithNewInstanceId) + { + // If restartWithNewInstanceId is True, the two instanceId should be different. + Assert.NotEqual(instanceId, restartInstanceId); + } + else + { + Assert.Equal(instanceId, restartInstanceId); + } + + HttpResponseMessage result = await HttpHelpers.InvokeHttpTrigger("RestartOrchestrator_Query_Tags", $"?id={restartInstanceId}"); + + // Verify that restarted orchestration instance contains tags. + Assert.Equal(HttpStatusCode.OK, result.StatusCode); + var content = await result.Content.ReadAsStringAsync(); + // Content: { "output": "...", "tags": { "testtag": "true" } } + Assert.Contains("output", content); + Assert.Contains("testtag", content); + } + + [Fact] + [Trait("PowerShell", "Skip")] // RestartAsync not yet implemented in PowerShell + [Trait("Java", "Skip")] // RestartAsync not yet implemented in Java + [Trait("Python", "Skip")] // RestartAsync not supported in Python + [Trait("Node", "Skip")] // RestartAsync not supported in Node + // Test that if we restart a instanceId that doesn't exist. We will throw ArgumentException exception. + public async Task RestartOrchestration_NonExistentInstanceId_ShouldReturnNotFound() + { + const string testInstanceId = "nonexistid"; + + // Test restarting with a non-existent instance ID + var restartPayload = new + { + InstanceId = testInstanceId, + RestartWithNewInstanceId = false + }; + + string jsonBody = JsonSerializer.Serialize(restartPayload); + + using HttpResponseMessage restartResponse = await HttpHelpers.InvokeHttpTriggerWithBody( + "RestartOrchestration_HttpRestartWithErrorHandling", jsonBody, "application/json"); + + string responseContent = await restartResponse.Content.ReadAsStringAsync(); + + // Verfity we weill return the right exception message. + Assert.Contains(fixture.functionLanguageLocalizer.GetLocalizedStringValue("RestartInvalidInstance.ErrorMessage", testInstanceId), responseContent); + } + + [Fact] + [Trait("PowerShell", "Skip")] // RestartAsync not yet implemented in PowerShell + [Trait("Java", "Skip")] // RestartAsync not yet implemented in Java + [Trait("Python", "Skip")] // RestartAsync not supported in Python + [Trait("Node", "Skip")] // RestartAsync not supported in Node + public async Task RestartOrchestration_NotCompletedOrchestrationWithRestartFalse_ShouldSucceed() + { + // Start a long-running orchestration + using HttpResponseMessage response = await HttpHelpers.InvokeHttpTrigger("RestarttOrchestration_HttpStart/LongOrchestrator"); + Assert.Equal(HttpStatusCode.Accepted, response.StatusCode); + string instanceId = await DurableHelpers.ParseInstanceIdAsync(response); + string statusQueryGetUri = await DurableHelpers.ParseStatusQueryGetUriAsync(response); + DurableHelpers.OrchestrationStatusDetails orchestrationDetails + = await DurableHelpers.GetRunningOrchestrationDetailsAsync(statusQueryGetUri); + DateTime createdTime1 = orchestrationDetails.CreatedTime; + + // Wait for the orchestration to be running + await DurableHelpers.WaitForOrchestrationStateAsync(statusQueryGetUri, "Running", 30); + + // Try to restart the running orchestration with restartWithNewInstanceId = false + var restartPayload = new + { + InstanceId = instanceId, + RestartWithNewInstanceId = false + }; + + string jsonBody = JsonSerializer.Serialize(restartPayload); + + // Restart the orchestrator with the same instance id) + using HttpResponseMessage restartResponse = await HttpHelpers.InvokeHttpTriggerWithBody( + "RestartOrchestration_HttpRestart", jsonBody, "application/json"); + Assert.Equal(HttpStatusCode.Accepted, restartResponse.StatusCode); + + string restartStatusQueryGetUri = await DurableHelpers.ParseStatusQueryGetUriAsync(restartResponse); + string restartInstanceId = await DurableHelpers.ParseInstanceIdAsync(restartResponse); + + await DurableHelpers.WaitForOrchestrationStateAsync(restartStatusQueryGetUri, "Running", 30); + DurableHelpers.OrchestrationStatusDetails restartOrchestrationDetails + = await DurableHelpers.GetRunningOrchestrationDetailsAsync(restartStatusQueryGetUri); + DateTime createdTime2 = restartOrchestrationDetails.CreatedTime; + + // Created time should be different. + Assert.NotEqual(createdTime1, createdTime2); + Assert.Equal(instanceId, restartInstanceId); + + // Clean up: terminate the long-running orchestration + using HttpResponseMessage terminateResponse = await HttpHelpers.InvokeHttpTrigger("TerminateInstance", $"?instanceId={instanceId}"); + Assert.Equal(HttpStatusCode.OK, terminateResponse.StatusCode); + } +} From b608832ce05020b1aaa0a01fa49bff63f45ed34c Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Wed, 11 Feb 2026 13:05:21 -0800 Subject: [PATCH 24/43] fixing the typoe --- src/WebJobs.Extensions.DurableTask/HttpApiHandler.cs | 2 +- test/Common/HttpApiHandlerTests.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/WebJobs.Extensions.DurableTask/HttpApiHandler.cs b/src/WebJobs.Extensions.DurableTask/HttpApiHandler.cs index 803d044b3..a4ec3664e 100644 --- a/src/WebJobs.Extensions.DurableTask/HttpApiHandler.cs +++ b/src/WebJobs.Extensions.DurableTask/HttpApiHandler.cs @@ -1081,7 +1081,7 @@ private async Task HandleRestartInstanceRequestAsync( } catch (OrchestrationAlreadyExistsException e) { - return request.CreateErrorResponse(HttpStatusCode.BadRequest, "A non-terminal instance with this intance ID already exists.", e); + return request.CreateErrorResponse(HttpStatusCode.BadRequest, "A non-terminal instance with this instance ID already exists.", e); } catch (JsonReaderException e) { diff --git a/test/Common/HttpApiHandlerTests.cs b/test/Common/HttpApiHandlerTests.cs index 7452fddf7..814ad6d86 100644 --- a/test/Common/HttpApiHandlerTests.cs +++ b/test/Common/HttpApiHandlerTests.cs @@ -1174,7 +1174,7 @@ public async Task RestartInstance_Returns_HTTP_400_On_Invalid_Existing_Instance( Assert.Equal(HttpStatusCode.BadRequest, actualResponse.StatusCode); var content = await actualResponse.Content.ReadAsStringAsync(); var error = JsonConvert.DeserializeObject(content); - Assert.Equal("A non-terminal instance with this intance ID already exists.", error["Message"].ToString()); + Assert.Equal("A non-terminal instance with this instance ID already exists.", error["Message"].ToString()); } [Theory] From 583c22ed93d24aa0db1f912c2a9f4caaecfbcfae Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Wed, 11 Feb 2026 13:52:09 -0800 Subject: [PATCH 25/43] added log assertions to the tests, removed the huge RestartOrchestrationTests diff, addressed more copilot comments --- .../Apps/BasicDotNetIsolated/HelloCities.cs | 4 +- test/e2e/Tests/Tests/DedupeStatusesTests.cs | 40 +- .../Tests/Tests/RestartOrchestrationTests.cs | 356 ++++++++++-------- 3 files changed, 230 insertions(+), 170 deletions(-) diff --git a/test/e2e/Apps/BasicDotNetIsolated/HelloCities.cs b/test/e2e/Apps/BasicDotNetIsolated/HelloCities.cs index 0b7900531..f04af2ab8 100644 --- a/test/e2e/Apps/BasicDotNetIsolated/HelloCities.cs +++ b/test/e2e/Apps/BasicDotNetIsolated/HelloCities.cs @@ -119,14 +119,14 @@ public static async Task StartOrchestration_DedupeStatuses( } catch (OrchestrationAlreadyExistsException ex) { - // Tests expect BadRequest for orchestration dedupe scenarios. + // Tests expect Conflict (409) for orchestration dedupe scenarios. HttpResponseData response = req.CreateResponse(HttpStatusCode.Conflict); await response.WriteStringAsync(ex.Message); return response; } catch (ArgumentException ex) { - // Tests expect BadRequest for orchestration dedupe scenarios. + // Tests expect BadRequest for invalid dedupe statuses HttpResponseData response = req.CreateResponse(HttpStatusCode.BadRequest); await response.WriteStringAsync(ex.Message); return response; diff --git a/test/e2e/Tests/Tests/DedupeStatusesTests.cs b/test/e2e/Tests/Tests/DedupeStatusesTests.cs index ccff86dca..c0ca001d8 100644 --- a/test/e2e/Tests/Tests/DedupeStatusesTests.cs +++ b/test/e2e/Tests/Tests/DedupeStatusesTests.cs @@ -206,7 +206,7 @@ public async Task StartOrchestration_WithInvalidDedupeStatuses_ThrowsArgumentExc "HelloCities", Guid.NewGuid().ToString(), expectedState: string.Empty, dedupeStatuses, expectedCode: HttpStatusCode.BadRequest); } - private static async Task StartAndWaitForState( + private async Task StartAndWaitForState( string orchestrationName, string instanceId, string expectedState, @@ -225,10 +225,20 @@ private static async Task StartAndWaitForState( Assert.Equal(HttpStatusCode.Accepted, response.StatusCode); string statusQueryGetUri = await DurableHelpers.ParseStatusQueryGetUriAsync(response); await DurableHelpers.WaitForOrchestrationStateAsync(statusQueryGetUri, expectedState, 60); + + if (expectedState != "Pending") + { + ClientOperationLogHelpers.AssertClientOperationLogExists( + () => this.fixture.TestLogs.CoreToolsLogs, + "StartOrchestration", + instanceId, + this.fixture.functionLanguageLocalizer.GetLanguageType()); + } + return response; } - private static async Task StartAndWaitForStateWithDedupeStatuses( + private async Task StartAndWaitForStateWithDedupeStatuses( string orchestrationName, string instanceId, string expectedState, @@ -252,22 +262,44 @@ private static async Task StartAndWaitForStateWithDedupeSta } string statusQueryGetUri = await DurableHelpers.ParseStatusQueryGetUriAsync(response); await DurableHelpers.WaitForOrchestrationStateAsync(statusQueryGetUri, expectedState, 60); + + if (expectedState != "Pending") + { + ClientOperationLogHelpers.AssertClientOperationLogExists( + () => this.fixture.TestLogs.CoreToolsLogs, + "StartOrchestration", + instanceId, + this.fixture.functionLanguageLocalizer.GetLanguageType()); + } + return response; } - private static async Task TerminateAndWaitForState(string instanceId, HttpResponseMessage startOrchestrationResponse) + private async Task TerminateAndWaitForState(string instanceId, HttpResponseMessage startOrchestrationResponse) { using HttpResponseMessage terminateResponse = await HttpHelpers.InvokeHttpTrigger("TerminateInstance", $"?instanceId={instanceId}"); Assert.Equal(HttpStatusCode.OK, terminateResponse.StatusCode); string statusQueryGetUri = await DurableHelpers.ParseStatusQueryGetUriAsync(startOrchestrationResponse); await DurableHelpers.WaitForOrchestrationStateAsync(statusQueryGetUri, "Terminated", 60); + + ClientOperationLogHelpers.AssertClientOperationLogExists( + () => this.fixture.TestLogs.CoreToolsLogs, + "Terminate", + instanceId, + this.fixture.functionLanguageLocalizer.GetLanguageType()); } - private static async Task SuspendAndWaitForState(string instanceId, HttpResponseMessage startOrchestrationResponse) + private async Task SuspendAndWaitForState(string instanceId, HttpResponseMessage startOrchestrationResponse) { using HttpResponseMessage suspendResponse = await HttpHelpers.InvokeHttpTrigger("SuspendInstance", $"?instanceId={instanceId}"); Assert.Equal(HttpStatusCode.OK, suspendResponse.StatusCode); string statusQueryGetUri = await DurableHelpers.ParseStatusQueryGetUriAsync(startOrchestrationResponse); await DurableHelpers.WaitForOrchestrationStateAsync(statusQueryGetUri, "Suspended", 60); + + ClientOperationLogHelpers.AssertClientOperationLogExists( + () => this.fixture.TestLogs.CoreToolsLogs, + "Suspend", + instanceId, + this.fixture.functionLanguageLocalizer.GetLanguageType()); } } diff --git a/test/e2e/Tests/Tests/RestartOrchestrationTests.cs b/test/e2e/Tests/Tests/RestartOrchestrationTests.cs index 5b4ed09c0..baf17e078 100644 --- a/test/e2e/Tests/Tests/RestartOrchestrationTests.cs +++ b/test/e2e/Tests/Tests/RestartOrchestrationTests.cs @@ -1,172 +1,200 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the MIT License. See License.txt in the project root for license information. - -using System.Net; -using System.Text.Json; -using Xunit; -using Xunit.Abstractions; - -namespace Microsoft.Azure.Durable.Tests.DotnetIsolatedE2E; - -[Collection(Constants.FunctionAppCollectionName)] -public class RestartOrchestrationTests -{ - private readonly FunctionAppFixture fixture; - private readonly ITestOutputHelper output; - - public RestartOrchestrationTests(FunctionAppFixture fixture, ITestOutputHelper testOutputHelper) - { - this.fixture = fixture; - this.fixture.TestLogs.UseTestLogger(testOutputHelper); - this.output = testOutputHelper; - } - - [Theory] - [InlineData(false)] - [InlineData(true)] - [Trait("PowerShell", "Skip")] // RestartAsync not yet implemented in PowerShell - [Trait("Java", "Skip")] // RestartAsync not yet implemented in Java - [Trait("Python", "Skip")] // RestartAsync not supported in Python - [Trait("Node", "Skip")] // RestartAsync not supported in Node - [Trait("MSSQL", "Skip")] // MSSQL doesn't support tags at ExecutionStarted Event. - // Test behavior of restartasync of durabletaskclient. - // When restart with a instanceid and startwithnewinstanceid is false, the orchestration should be restarted with the same instance id. - // When restart with a instanceid and startwithnewinstanceid is true, the orchestration should be restarted with a new instance id. - public async Task RestartOrchestration_CreatedTimeAndOutputChange(bool restartWithNewInstanceId) - { - // Start the orchestration - using HttpResponseMessage response = await HttpHelpers.InvokeHttpTrigger("RestarttOrchestration_HttpStart/SimpleOrchestrator"); - Assert.Equal(HttpStatusCode.Accepted, response.StatusCode); - string statusQueryGetUri = await DurableHelpers.ParseStatusQueryGetUriAsync(response); +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System.Net; +using System.Text.Json; +using Xunit; +using Xunit.Abstractions; + +namespace Microsoft.Azure.Durable.Tests.DotnetIsolatedE2E; + +[Collection(Constants.FunctionAppCollectionName)] +public class RestartOrchestrationTests +{ + private readonly FunctionAppFixture fixture; + private readonly ITestOutputHelper output; + + public RestartOrchestrationTests(FunctionAppFixture fixture, ITestOutputHelper testOutputHelper) + { + this.fixture = fixture; + this.fixture.TestLogs.UseTestLogger(testOutputHelper); + this.output = testOutputHelper; + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + [Trait("PowerShell", "Skip")] // RestartAsync not yet implemented in PowerShell + [Trait("Java", "Skip")] // RestartAsync not yet implemented in Java + [Trait("Python", "Skip")] // RestartAsync not supported in Python + [Trait("Node", "Skip")] // RestartAsync not supported in Node + [Trait("MSSQL", "Skip")] // MSSQL doesn't support tags at ExecutionStarted Event. + // Test behavior of restartasync of durabletaskclient. + // When restart with a instanceid and startwithnewinstanceid is false, the orchestration should be restarted with the same instance id. + // When restart with a instanceid and startwithnewinstanceid is true, the orchestration should be restarted with a new instance id. + public async Task RestartOrchestration_CreatedTimeAndOutputChange(bool restartWithNewInstanceId) + { + // Start the orchestration + using HttpResponseMessage response = await HttpHelpers.InvokeHttpTrigger("RestarttOrchestration_HttpStart/SimpleOrchestrator"); + Assert.Equal(HttpStatusCode.Accepted, response.StatusCode); + string statusQueryGetUri = await DurableHelpers.ParseStatusQueryGetUriAsync(response); string instanceId = await DurableHelpers.ParseInstanceIdAsync(response); - await DurableHelpers.WaitForOrchestrationStateAsync(statusQueryGetUri, "Completed", 30); - var orchestrationDetails = await DurableHelpers.GetRunningOrchestrationDetailsAsync(statusQueryGetUri); - string output1 = orchestrationDetails.Output; - DateTime createdTime1 = orchestrationDetails.CreatedTime; - - // best practice to wait for 1 seconds before restarting orchestration to avoid race condition. - await Task.Delay(1000); - - var restartPayload = new { - InstanceId = instanceId, - RestartWithNewInstanceId = restartWithNewInstanceId - }; - - string jsonBody = JsonSerializer.Serialize(restartPayload); - - // Restart the orchestrator with the same instance id) - using HttpResponseMessage restartResponse = await HttpHelpers.InvokeHttpTriggerWithBody( - "RestartOrchestration_HttpRestart", jsonBody, "application/json"); - Assert.Equal(HttpStatusCode.Accepted, restartResponse.StatusCode); - string restartStatusQueryGetUri = await DurableHelpers.ParseStatusQueryGetUriAsync(restartResponse); + // Verify that the ClientOperationReceived log was emitted with a FunctionInvocationId + ClientOperationLogHelpers.AssertClientOperationLogExists( + () => this.fixture.TestLogs.CoreToolsLogs, + "StartOrchestration", + instanceId, + this.fixture.functionLanguageLocalizer.GetLanguageType()); + + await DurableHelpers.WaitForOrchestrationStateAsync(statusQueryGetUri, "Completed", 30); + var orchestrationDetails = await DurableHelpers.GetRunningOrchestrationDetailsAsync(statusQueryGetUri); + string output1 = orchestrationDetails.Output; + DateTime createdTime1 = orchestrationDetails.CreatedTime; + + // best practice to wait for 1 seconds before restarting orchestration to avoid race condition. + await Task.Delay(1000); + + var restartPayload = new { + InstanceId = instanceId, + RestartWithNewInstanceId = restartWithNewInstanceId + }; + + string jsonBody = JsonSerializer.Serialize(restartPayload); + + // Restart the orchestrator with the same instance id) + using HttpResponseMessage restartResponse = await HttpHelpers.InvokeHttpTriggerWithBody( + "RestartOrchestration_HttpRestart", jsonBody, "application/json"); + Assert.Equal(HttpStatusCode.Accepted, restartResponse.StatusCode); + string restartStatusQueryGetUri = await DurableHelpers.ParseStatusQueryGetUriAsync(restartResponse); string restartInstanceId = await DurableHelpers.ParseInstanceIdAsync(restartResponse); - await DurableHelpers.WaitForOrchestrationStateAsync(restartStatusQueryGetUri, "Completed", 30); - var restartOrchestrationDetails = await DurableHelpers.GetRunningOrchestrationDetailsAsync(restartStatusQueryGetUri); - string output2 = restartOrchestrationDetails.Output; - DateTime createdTime2 = restartOrchestrationDetails.CreatedTime; - - // The outputs should be the same as input is same. - Assert.Equal(output1, output2); - // Created time should be different. - Assert.NotEqual(createdTime1, createdTime2); - - if (restartWithNewInstanceId) - { - // If restartWithNewInstanceId is True, the two instanceId should be different. - Assert.NotEqual(instanceId, restartInstanceId); - } - else - { - Assert.Equal(instanceId, restartInstanceId); - } - - HttpResponseMessage result = await HttpHelpers.InvokeHttpTrigger("RestartOrchestrator_Query_Tags", $"?id={restartInstanceId}"); - - // Verify that restarted orchestration instance contains tags. - Assert.Equal(HttpStatusCode.OK, result.StatusCode); - var content = await result.Content.ReadAsStringAsync(); - // Content: { "output": "...", "tags": { "testtag": "true" } } - Assert.Contains("output", content); - Assert.Contains("testtag", content); - } - - [Fact] - [Trait("PowerShell", "Skip")] // RestartAsync not yet implemented in PowerShell - [Trait("Java", "Skip")] // RestartAsync not yet implemented in Java - [Trait("Python", "Skip")] // RestartAsync not supported in Python - [Trait("Node", "Skip")] // RestartAsync not supported in Node - // Test that if we restart a instanceId that doesn't exist. We will throw ArgumentException exception. - public async Task RestartOrchestration_NonExistentInstanceId_ShouldReturnNotFound() - { - const string testInstanceId = "nonexistid"; - - // Test restarting with a non-existent instance ID - var restartPayload = new - { - InstanceId = testInstanceId, - RestartWithNewInstanceId = false - }; - - string jsonBody = JsonSerializer.Serialize(restartPayload); - - using HttpResponseMessage restartResponse = await HttpHelpers.InvokeHttpTriggerWithBody( - "RestartOrchestration_HttpRestartWithErrorHandling", jsonBody, "application/json"); - - string responseContent = await restartResponse.Content.ReadAsStringAsync(); - - // Verfity we weill return the right exception message. - Assert.Contains(fixture.functionLanguageLocalizer.GetLocalizedStringValue("RestartInvalidInstance.ErrorMessage", testInstanceId), responseContent); - } - - [Fact] - [Trait("PowerShell", "Skip")] // RestartAsync not yet implemented in PowerShell - [Trait("Java", "Skip")] // RestartAsync not yet implemented in Java - [Trait("Python", "Skip")] // RestartAsync not supported in Python - [Trait("Node", "Skip")] // RestartAsync not supported in Node - public async Task RestartOrchestration_NotCompletedOrchestrationWithRestartFalse_ShouldSucceed() - { - // Start a long-running orchestration - using HttpResponseMessage response = await HttpHelpers.InvokeHttpTrigger("RestarttOrchestration_HttpStart/LongOrchestrator"); - Assert.Equal(HttpStatusCode.Accepted, response.StatusCode); - string instanceId = await DurableHelpers.ParseInstanceIdAsync(response); - string statusQueryGetUri = await DurableHelpers.ParseStatusQueryGetUriAsync(response); - DurableHelpers.OrchestrationStatusDetails orchestrationDetails - = await DurableHelpers.GetRunningOrchestrationDetailsAsync(statusQueryGetUri); + // Verify that the ClientOperationReceived log was emitted with a FunctionInvocationId + ClientOperationLogHelpers.AssertClientOperationLogExists( + () => this.fixture.TestLogs.CoreToolsLogs, + "Restart", + instanceId, + this.fixture.functionLanguageLocalizer.GetLanguageType()); + + await DurableHelpers.WaitForOrchestrationStateAsync(restartStatusQueryGetUri, "Completed", 30); + var restartOrchestrationDetails = await DurableHelpers.GetRunningOrchestrationDetailsAsync(restartStatusQueryGetUri); + string output2 = restartOrchestrationDetails.Output; + DateTime createdTime2 = restartOrchestrationDetails.CreatedTime; + + // The outputs should be the same as input is same. + Assert.Equal(output1, output2); + // Created time should be different. + Assert.NotEqual(createdTime1, createdTime2); + + if (restartWithNewInstanceId) + { + // If restartWithNewInstanceId is True, the two instanceId should be different. + Assert.NotEqual(instanceId, restartInstanceId); + } + else + { + Assert.Equal(instanceId, restartInstanceId); + } + + HttpResponseMessage result = await HttpHelpers.InvokeHttpTrigger("RestartOrchestrator_Query_Tags", $"?id={restartInstanceId}"); + + // Verify that restarted orchestration instance contains tags. + Assert.Equal(HttpStatusCode.OK, result.StatusCode); + var content = await result.Content.ReadAsStringAsync(); + // Content: { "output": "...", "tags": { "testtag": "true" } } + Assert.Contains("output", content); + Assert.Contains("testtag", content); + } + + [Fact] + [Trait("PowerShell", "Skip")] // RestartAsync not yet implemented in PowerShell + [Trait("Java", "Skip")] // RestartAsync not yet implemented in Java + [Trait("Python", "Skip")] // RestartAsync not supported in Python + [Trait("Node", "Skip")] // RestartAsync not supported in Node + // Test that if we restart a instanceId that doesn't exist. We will throw ArgumentException exception. + public async Task RestartOrchestration_NonExistentInstanceId_ShouldReturnNotFound() + { + const string testInstanceId = "nonexistid"; + + // Test restarting with a non-existent instance ID + var restartPayload = new + { + InstanceId = testInstanceId, + RestartWithNewInstanceId = false + }; + + string jsonBody = JsonSerializer.Serialize(restartPayload); + + using HttpResponseMessage restartResponse = await HttpHelpers.InvokeHttpTriggerWithBody( + "RestartOrchestration_HttpRestartWithErrorHandling", jsonBody, "application/json"); + + string responseContent = await restartResponse.Content.ReadAsStringAsync(); + + // Verfity we weill return the right exception message. + Assert.Contains(fixture.functionLanguageLocalizer.GetLocalizedStringValue("RestartInvalidInstance.ErrorMessage", testInstanceId), responseContent); + } + + [Fact] + [Trait("PowerShell", "Skip")] // RestartAsync not yet implemented in PowerShell + [Trait("Java", "Skip")] // RestartAsync not yet implemented in Java + [Trait("Python", "Skip")] // RestartAsync not supported in Python + [Trait("Node", "Skip")] // RestartAsync not supported in Node + public async Task RestartOrchestration_NotCompletedOrchestrationWithRestartFalse_ShouldSucceed() + { + // Start a long-running orchestration + using HttpResponseMessage response = await HttpHelpers.InvokeHttpTrigger("RestarttOrchestration_HttpStart/LongOrchestrator"); + Assert.Equal(HttpStatusCode.Accepted, response.StatusCode); + string instanceId = await DurableHelpers.ParseInstanceIdAsync(response); + string statusQueryGetUri = await DurableHelpers.ParseStatusQueryGetUriAsync(response); + DurableHelpers.OrchestrationStatusDetails orchestrationDetails + = await DurableHelpers.GetRunningOrchestrationDetailsAsync(statusQueryGetUri); DateTime createdTime1 = orchestrationDetails.CreatedTime; - // Wait for the orchestration to be running - await DurableHelpers.WaitForOrchestrationStateAsync(statusQueryGetUri, "Running", 30); - - // Try to restart the running orchestration with restartWithNewInstanceId = false - var restartPayload = new - { - InstanceId = instanceId, - RestartWithNewInstanceId = false - }; - - string jsonBody = JsonSerializer.Serialize(restartPayload); - - // Restart the orchestrator with the same instance id) - using HttpResponseMessage restartResponse = await HttpHelpers.InvokeHttpTriggerWithBody( - "RestartOrchestration_HttpRestart", jsonBody, "application/json"); - Assert.Equal(HttpStatusCode.Accepted, restartResponse.StatusCode); - - string restartStatusQueryGetUri = await DurableHelpers.ParseStatusQueryGetUriAsync(restartResponse); + // Verify that the ClientOperationReceived log was emitted with a FunctionInvocationId + ClientOperationLogHelpers.AssertClientOperationLogExists( + () => this.fixture.TestLogs.CoreToolsLogs, + "StartOrchestration", + instanceId, + this.fixture.functionLanguageLocalizer.GetLanguageType()); + + // Wait for the orchestration to be running + await DurableHelpers.WaitForOrchestrationStateAsync(statusQueryGetUri, "Running", 30); + + // Try to restart the running orchestration with restartWithNewInstanceId = false + var restartPayload = new + { + InstanceId = instanceId, + RestartWithNewInstanceId = false + }; + + string jsonBody = JsonSerializer.Serialize(restartPayload); + + // Restart the orchestrator with the same instance id) + using HttpResponseMessage restartResponse = await HttpHelpers.InvokeHttpTriggerWithBody( + "RestartOrchestration_HttpRestart", jsonBody, "application/json"); + Assert.Equal(HttpStatusCode.Accepted, restartResponse.StatusCode); + + string restartStatusQueryGetUri = await DurableHelpers.ParseStatusQueryGetUriAsync(restartResponse); string restartInstanceId = await DurableHelpers.ParseInstanceIdAsync(restartResponse); - await DurableHelpers.WaitForOrchestrationStateAsync(restartStatusQueryGetUri, "Running", 30); - DurableHelpers.OrchestrationStatusDetails restartOrchestrationDetails - = await DurableHelpers.GetRunningOrchestrationDetailsAsync(restartStatusQueryGetUri); - DateTime createdTime2 = restartOrchestrationDetails.CreatedTime; - - // Created time should be different. - Assert.NotEqual(createdTime1, createdTime2); - Assert.Equal(instanceId, restartInstanceId); - - // Clean up: terminate the long-running orchestration - using HttpResponseMessage terminateResponse = await HttpHelpers.InvokeHttpTrigger("TerminateInstance", $"?instanceId={instanceId}"); - Assert.Equal(HttpStatusCode.OK, terminateResponse.StatusCode); - } -} + // Verify that the ClientOperationReceived log was emitted with a FunctionInvocationId + ClientOperationLogHelpers.AssertClientOperationLogExists( + () => this.fixture.TestLogs.CoreToolsLogs, + "Restart", + instanceId, + this.fixture.functionLanguageLocalizer.GetLanguageType()); + + await DurableHelpers.WaitForOrchestrationStateAsync(restartStatusQueryGetUri, "Running", 30); + DurableHelpers.OrchestrationStatusDetails restartOrchestrationDetails + = await DurableHelpers.GetRunningOrchestrationDetailsAsync(restartStatusQueryGetUri); + DateTime createdTime2 = restartOrchestrationDetails.CreatedTime; + + // Created time should be different. + Assert.NotEqual(createdTime1, createdTime2); + Assert.Equal(instanceId, restartInstanceId); + + // Clean up: terminate the long-running orchestration + using HttpResponseMessage terminateResponse = await HttpHelpers.InvokeHttpTrigger("TerminateInstance", $"?instanceId={instanceId}"); + Assert.Equal(HttpStatusCode.OK, terminateResponse.StatusCode); + } +} From d530408d8cd816fbcb495aff84ab7cbc27971994 Mon Sep 17 00:00:00 2001 From: sophiatev <38052607+sophiatev@users.noreply.github.com> Date: Wed, 11 Feb 2026 13:52:34 -0800 Subject: [PATCH 26/43] Update test/e2e/Apps/BasicJava/src/main/java/com/function/HelloCities.java Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .../BasicJava/src/main/java/com/function/HelloCities.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/e2e/Apps/BasicJava/src/main/java/com/function/HelloCities.java b/test/e2e/Apps/BasicJava/src/main/java/com/function/HelloCities.java index 133a3f192..9c1bfc5eb 100644 --- a/test/e2e/Apps/BasicJava/src/main/java/com/function/HelloCities.java +++ b/test/e2e/Apps/BasicJava/src/main/java/com/function/HelloCities.java @@ -52,8 +52,9 @@ public HttpResponseMessage startOrchestration( final ExecutionContext context) { DurableTaskClient client = durableContext.getClient(); String orchestrationName = request.getQueryParameters().get("orchestrationName"); - String instanceId = request.getQueryParameters().get("instanceId"); - instanceId = client.scheduleNewOrchestrationInstance(orchestrationName, "", instanceId); + NewOrchestrationInstanceOptions startOptions = new NewOrchestrationInstanceOptions(); + startOptions.setInstanceId(request.getQueryParameters().get("instanceId")); + String instanceId = client.scheduleNewOrchestrationInstance(orchestrationName, startOptions); context.getLogger().info("Started orchestration with ID = '" + instanceId + "'."); return durableContext.createCheckStatusResponse(request, instanceId); } From 2ea38905429f8562ac8491e230ad3deb0b909ba8 Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Thu, 19 Feb 2026 17:49:19 -0800 Subject: [PATCH 27/43] fixing the build warnings and addressing PR comments --- .../DurabilityProvider.cs | 23 ++++++++++++------- .../HttpApiHandler.cs | 2 +- test/Common/DurableTaskEndToEndTests.cs | 2 +- 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/src/WebJobs.Extensions.DurableTask/DurabilityProvider.cs b/src/WebJobs.Extensions.DurableTask/DurabilityProvider.cs index bda44a546..e140a6698 100644 --- a/src/WebJobs.Extensions.DurableTask/DurabilityProvider.cs +++ b/src/WebJobs.Extensions.DurableTask/DurabilityProvider.cs @@ -11,6 +11,7 @@ using DurableTask.Core.Exceptions; using DurableTask.Core.History; using DurableTask.Core.Query; +using Grpc.Core; using Microsoft.Azure.WebJobs.Host.Scale; using Newtonsoft.Json; using Newtonsoft.Json.Linq; @@ -430,10 +431,10 @@ public Task CreateTaskOrchestrationAsync(TaskMessage creationMessage) /// to be reusable. In this case, an existing orchestration with that running status would be terminated, but the creation of the new orchestration /// would immediately fail due to the existing orchestration now having status . /// - public Task CreateTaskOrchestrationAsync(TaskMessage creationMessage, OrchestrationStatus[] dedupeStatuses) + public async Task CreateTaskOrchestrationAsync(TaskMessage creationMessage, OrchestrationStatus[] dedupeStatuses) { using var timeoutCts = new CancellationTokenSource(TimeSpan.FromSeconds(this.orchestrationCreationRequestTimeoutInSeconds)); - return this.CreateTaskOrchestrationAsync(creationMessage, dedupeStatuses, timeoutCts.Token); + await this.CreateTaskOrchestrationAsync(creationMessage, dedupeStatuses, timeoutCts.Token); } /// @@ -732,12 +733,18 @@ bool IsRunning(OrchestrationStatus status) => // Check for cancellation before attempting to terminate the orchestration cancellationToken.ThrowIfCancellationRequested(); - await this.ForceTerminateTaskOrchestrationAsync( - instanceId, - $"A new instance creation request has been issued for instance {instanceId} which currently has status " + - $"{orchestrationState.OrchestrationStatus}. Since the dedupe statuses of the creation request, " + - $"{(dedupeStatuses == null ? "[]" : string.Join(", ", dedupeStatuses))}, do not contain the orchestration's " + - $"status, the orchestration has been terminated and a new instance with the same instance ID will be created."); + string dedupeStatusesDescription = dedupeStatuses == null + ? "null (all statuses reusable)" + : dedupeStatuses.Length == 0 + ? "[] (all statuses reusable)" + : $"[{string.Join(", ", dedupeStatuses)}]"; + + string terminationReason = $"A new instance creation request has been issued for instance {instanceId} which " + + $"currently has status {orchestrationState.OrchestrationStatus}. Since the dedupe statuses of the creation request, " + + $"{dedupeStatusesDescription}, do not contain the orchestration's status, the orchestration has been terminated " + + $"and a new instance with the same instance ID will be created."; + + await this.ForceTerminateTaskOrchestrationAsync(instanceId, terminationReason); await this.WaitForOrchestrationAsync( instanceId, diff --git a/src/WebJobs.Extensions.DurableTask/HttpApiHandler.cs b/src/WebJobs.Extensions.DurableTask/HttpApiHandler.cs index a4ec3664e..8170913b3 100644 --- a/src/WebJobs.Extensions.DurableTask/HttpApiHandler.cs +++ b/src/WebJobs.Extensions.DurableTask/HttpApiHandler.cs @@ -1081,7 +1081,7 @@ private async Task HandleRestartInstanceRequestAsync( } catch (OrchestrationAlreadyExistsException e) { - return request.CreateErrorResponse(HttpStatusCode.BadRequest, "A non-terminal instance with this instance ID already exists.", e); + return request.CreateErrorResponse(HttpStatusCode.Conflict, "A non-terminal instance with this instance ID already exists.", e); } catch (JsonReaderException e) { diff --git a/test/Common/DurableTaskEndToEndTests.cs b/test/Common/DurableTaskEndToEndTests.cs index a91b86178..f98d1557e 100644 --- a/test/Common/DurableTaskEndToEndTests.cs +++ b/test/Common/DurableTaskEndToEndTests.cs @@ -6169,7 +6169,7 @@ private async Task OverridableStates_TerminalStatusesAlwaysReusable( TestDurableClient client; client = await host.StartOrchestratorAsync( - terminalStatus == OrchestrationRuntimeStatus.Failed + terminalStatus == OrchestrationRuntimeStatus.Failed ? nameof(TestOrchestrations.ThrowOrchestrator) : nameof(TestOrchestrations.Counter), terminalStatus == OrchestrationRuntimeStatus.Failed ? string.Empty : initialValue, this.output, From cda3ae052a661cf294d8c478ac4a136d63a7b295 Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Thu, 19 Feb 2026 18:04:27 -0800 Subject: [PATCH 28/43] removing the version specified in one of the package references in VSSample.csproj since we use CPM --- samples/precompiled/VSSample.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/precompiled/VSSample.csproj b/samples/precompiled/VSSample.csproj index 4dca46892..6aa1f0c42 100644 --- a/samples/precompiled/VSSample.csproj +++ b/samples/precompiled/VSSample.csproj @@ -12,7 +12,7 @@ - + From 5f1a596ea30fb4fb4c9e8d61a684d2c9597f530b Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Thu, 19 Feb 2026 21:07:55 -0800 Subject: [PATCH 29/43] further fixing the nuget build errors --- samples/Directory.Packages.props | 1 + 1 file changed, 1 insertion(+) diff --git a/samples/Directory.Packages.props b/samples/Directory.Packages.props index a64b43df2..0084c8f4d 100644 --- a/samples/Directory.Packages.props +++ b/samples/Directory.Packages.props @@ -16,6 +16,7 @@ + From 2c0af1e01893a2a15812e02a56ba82a50ae99dda Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Thu, 19 Feb 2026 21:12:04 -0800 Subject: [PATCH 30/43] continuing the nuget attempts, addressing some copilot comments --- samples/csx/extensions.csproj | 2 +- src/WebJobs.Extensions.DurableTask/DurabilityProvider.cs | 1 - test/Common/HttpApiHandlerTests.cs | 4 ++-- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/samples/csx/extensions.csproj b/samples/csx/extensions.csproj index 2d6316f02..77408b036 100644 --- a/samples/csx/extensions.csproj +++ b/samples/csx/extensions.csproj @@ -12,6 +12,6 @@ - + \ No newline at end of file diff --git a/src/WebJobs.Extensions.DurableTask/DurabilityProvider.cs b/src/WebJobs.Extensions.DurableTask/DurabilityProvider.cs index e140a6698..1a35cd425 100644 --- a/src/WebJobs.Extensions.DurableTask/DurabilityProvider.cs +++ b/src/WebJobs.Extensions.DurableTask/DurabilityProvider.cs @@ -11,7 +11,6 @@ using DurableTask.Core.Exceptions; using DurableTask.Core.History; using DurableTask.Core.Query; -using Grpc.Core; using Microsoft.Azure.WebJobs.Host.Scale; using Newtonsoft.Json; using Newtonsoft.Json.Linq; diff --git a/test/Common/HttpApiHandlerTests.cs b/test/Common/HttpApiHandlerTests.cs index 814ad6d86..4695c6bab 100644 --- a/test/Common/HttpApiHandlerTests.cs +++ b/test/Common/HttpApiHandlerTests.cs @@ -1150,7 +1150,7 @@ public async Task RestartInstance_Returns_HTTP_400_On_Invalid_InstanceId() [Fact] [Trait("Category", PlatformSpecificHelpers.TestCategory)] - public async Task RestartInstance_Returns_HTTP_400_On_Invalid_Existing_Instance() + public async Task RestartInstance_Returns_HTTP_409_On_Invalid_Existing_Instance() { string testBadInstanceId = Guid.NewGuid().ToString("N"); @@ -1171,7 +1171,7 @@ public async Task RestartInstance_Returns_HTTP_400_On_Invalid_Existing_Instance( var httpApiHandler = new ExtendedHttpApiHandler(clientMock.Object); var actualResponse = await httpApiHandler.HandleRequestAsync(testRequest, CancellationToken.None); - Assert.Equal(HttpStatusCode.BadRequest, actualResponse.StatusCode); + Assert.Equal(HttpStatusCode.Conflict, actualResponse.StatusCode); var content = await actualResponse.Content.ReadAsStringAsync(); var error = JsonConvert.DeserializeObject(content); Assert.Equal("A non-terminal instance with this instance ID already exists.", error["Message"].ToString()); From 8959fcf6c0fab9d0447a7f3e6937451db85501e2 Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Thu, 19 Feb 2026 21:31:11 -0800 Subject: [PATCH 31/43] fixed the failing test --- test/Common/HttpApiHandlerTests.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/Common/HttpApiHandlerTests.cs b/test/Common/HttpApiHandlerTests.cs index 4695c6bab..586f297ec 100644 --- a/test/Common/HttpApiHandlerTests.cs +++ b/test/Common/HttpApiHandlerTests.cs @@ -1380,8 +1380,7 @@ public async Task StartNewInstance_Returns_HTTP_400_On_Missing_Function() Assert.Equal(HttpStatusCode.BadRequest, actualResponse.StatusCode); var content = await actualResponse.Content.ReadAsStringAsync(); var error = JsonConvert.DeserializeObject(content); - Assert.Equal("One or more of the arguments submitted is incorrect", error["Message"].ToString()); - Assert.Equal(exceptionMessage, error["ExceptionMessage"].ToString()); + Assert.Equal(exceptionMessage, error["Message"].ToString()); } [Fact] From 513376df01526d13bd8f64712aa2a200eb5da3fa Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Thu, 26 Feb 2026 14:28:17 -0800 Subject: [PATCH 32/43] updated .NET SDK dependencies --- Directory.Packages.props | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index 32dce92dc..4acd9fabc 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -27,9 +27,9 @@ - - - + + + From d478db508a57cf665225bb8ca43d5871184384f7 Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Fri, 27 Feb 2026 11:31:44 -0800 Subject: [PATCH 33/43] attempting to fix the e2e dts and mssql errors --- .../DurabilityProvider.cs | 5 +++-- .../BasicDotNetIsolated/LargeOutputOrchestrator.cs | 4 ++++ .../java/com/function/LargeOutputOrchestrator.java | 4 ++++ .../src/functions/LargeOutputOrchestrator.ts | 3 +++ .../LargeOutputOrchestrator/run.ps1 | 4 ++++ .../Apps/BasicPython/large_output_orchestrator.py | 2 ++ test/e2e/Tests/Tests/DedupeStatusesTests.cs | 14 +++++++------- 7 files changed, 27 insertions(+), 9 deletions(-) diff --git a/src/WebJobs.Extensions.DurableTask/DurabilityProvider.cs b/src/WebJobs.Extensions.DurableTask/DurabilityProvider.cs index 1a35cd425..5d27d59d8 100644 --- a/src/WebJobs.Extensions.DurableTask/DurabilityProvider.cs +++ b/src/WebJobs.Extensions.DurableTask/DurabilityProvider.cs @@ -446,7 +446,8 @@ public async Task CreateTaskOrchestrationAsync(TaskMessage creationMessage, Orch /// If the array contains all of the running statuses (, , /// and ), then only terminal statuses can be reused. /// If at least one of these statuses is not included in the array, then if an instance with that status is found, it will first be terminated - /// before a new orchestration is created. + /// before a new orchestration is created. This method will wait for the instance to reach a terminal status for a maximum of one hour or + /// until the is invoked, whichever occurs first. /// The cancellation token used to cancel waiting for an existing instance to terminate in the case that a /// non-terminal instance is found whose runtime status is not included in . /// A task that completes when the creation message for the task orchestration instance is enqueued. @@ -748,7 +749,7 @@ bool IsRunning(OrchestrationStatus status) => await this.WaitForOrchestrationAsync( instanceId, orchestrationState.OrchestrationInstance.ExecutionId, - TimeSpan.MaxValue, + TimeSpan.FromHours(1), cancellationToken); } } diff --git a/test/e2e/Apps/BasicDotNetIsolated/LargeOutputOrchestrator.cs b/test/e2e/Apps/BasicDotNetIsolated/LargeOutputOrchestrator.cs index d8c927948..a4ec889f7 100644 --- a/test/e2e/Apps/BasicDotNetIsolated/LargeOutputOrchestrator.cs +++ b/test/e2e/Apps/BasicDotNetIsolated/LargeOutputOrchestrator.cs @@ -19,6 +19,10 @@ public static async Task> RunOrchestrator( { ILogger logger = context.CreateReplaySafeLogger(nameof(LargeOutputOrchestrator)); int sizeInKB = context.GetInput(); + if (sizeInKB <= 0) + { + throw new ArgumentOutOfRangeException(nameof(sizeInKB)); + } logger.LogInformation("Saying hello."); var outputs = new List(); diff --git a/test/e2e/Apps/BasicJava/src/main/java/com/function/LargeOutputOrchestrator.java b/test/e2e/Apps/BasicJava/src/main/java/com/function/LargeOutputOrchestrator.java index ec81fc47e..1931cc256 100644 --- a/test/e2e/Apps/BasicJava/src/main/java/com/function/LargeOutputOrchestrator.java +++ b/test/e2e/Apps/BasicJava/src/main/java/com/function/LargeOutputOrchestrator.java @@ -24,6 +24,10 @@ public List runOrchestrator( final ExecutionContext context) { int sizeInKB = ctx.getInput(Integer.class); + if (sizeInKB <= 0) { + throw new IllegalArgumentException("sizeInKB must be a positive integer."); + } + context.getLogger().info("Saying hello."); List outputs = new ArrayList<>(); diff --git a/test/e2e/Apps/BasicNode/src/functions/LargeOutputOrchestrator.ts b/test/e2e/Apps/BasicNode/src/functions/LargeOutputOrchestrator.ts index bec01eba9..ddbbefe58 100644 --- a/test/e2e/Apps/BasicNode/src/functions/LargeOutputOrchestrator.ts +++ b/test/e2e/Apps/BasicNode/src/functions/LargeOutputOrchestrator.ts @@ -13,6 +13,9 @@ function generateLargeString(sizeInKB: number): string { // Orchestration const LargeOutputOrchestrator: OrchestrationHandler = function* (context: OrchestrationContext) { const sizeInKB = context.df.getInput(); + if (sizeInKB <= 0) { + throw new Error('sizeInKB must be a positive integer.'); + } context.log('Saying hello.'); const outputs: any[] = []; const r_1 = yield context.df.callActivity('large_output_say_hello', 'Tokyo'); diff --git a/test/e2e/Apps/BasicPowerShell/LargeOutputOrchestrator/run.ps1 b/test/e2e/Apps/BasicPowerShell/LargeOutputOrchestrator/run.ps1 index abf7cd1ef..cc0236eb6 100644 --- a/test/e2e/Apps/BasicPowerShell/LargeOutputOrchestrator/run.ps1 +++ b/test/e2e/Apps/BasicPowerShell/LargeOutputOrchestrator/run.ps1 @@ -7,6 +7,10 @@ param($Context) $sizeInKB = [int]$Context.Input +if ($sizeInKB -le 0) { + [System.ArgumentOutOfRangeException]::new("sizeInKB") +} + Write-Information "Saying hello." $outputs = @() diff --git a/test/e2e/Apps/BasicPython/large_output_orchestrator.py b/test/e2e/Apps/BasicPython/large_output_orchestrator.py index 3388df00f..d39d2d7c6 100644 --- a/test/e2e/Apps/BasicPython/large_output_orchestrator.py +++ b/test/e2e/Apps/BasicPython/large_output_orchestrator.py @@ -19,6 +19,8 @@ def generate_large_string(size_in_kb: int) -> str: @bp.orchestration_trigger(context_name="context", orchestration="LargeOutputOrchestrator") def large_output_orchestrator(context: df.DurableOrchestrationContext): size_in_kb = context.get_input() + if (size_in_kb is None or size_in_kb<= 0): + raise ValueError("size_in_kb must be a positive integer.") logging.info("Saying hello.") outputs = [] r_1 = yield context.call_activity("large_output_say_hello", "Tokyo") diff --git a/test/e2e/Tests/Tests/DedupeStatusesTests.cs b/test/e2e/Tests/Tests/DedupeStatusesTests.cs index c0ca001d8..a02bca9b0 100644 --- a/test/e2e/Tests/Tests/DedupeStatusesTests.cs +++ b/test/e2e/Tests/Tests/DedupeStatusesTests.cs @@ -32,13 +32,12 @@ public async Task CanStartOrchestration_WithSameId_ForAllStatuses_ForEmptyDedupe "HelloCities", completedInstanceId, "Completed"); // Failed + // This invocation will fail because the "LargeOutputOrchestrator" expects a non-zero input, but we provide none string failedInstanceId = Guid.NewGuid().ToString(); using HttpResponseMessage startFailedResponseFirstAttempt = await StartAndWaitForState( - "RethrowActivityException", failedInstanceId, "Failed"); - // Invoking this same orchestration with the same instance ID will cause it to complete successfully on the second attempt, - // hence we look for a "Completed" status instead + "LargeOutputOrchestrator", failedInstanceId, "Failed"); using HttpResponseMessage startFailedResponseSecondAttempt = await StartAndWaitForState( - "RethrowActivityException", failedInstanceId, "Completed"); + "LargeOutputOrchestrator", failedInstanceId, "Failed"); // Terminated if (this.fixture.functionLanguageLocalizer.GetLanguageType() != LanguageType.Java @@ -136,13 +135,14 @@ public async Task StartOrchestration_WithSameId_FailsIfExistingStatus_InDedupeSt } // Failed + // This invocation will fail because the "LargeOutputOrchestrator" expects a non-zero input, but we provide none string failedInstanceId = Guid.NewGuid().ToString(); using HttpResponseMessage startFailedResponseFirstAttempt = await StartAndWaitForStateWithDedupeStatuses( - "RethrowActivityException", failedInstanceId, "Failed", dedupeStatuses); + "LargeOutputOrchestrator", failedInstanceId, "Failed", dedupeStatuses); using HttpResponseMessage startFailedResponseSecondAttempt = await StartAndWaitForStateWithDedupeStatuses( - "RethrowActivityException", + "LargeOutputOrchestrator", failedInstanceId, - "Completed", + "Failed", dedupeStatuses, expectedCode: dedupeStatuses.Contains("Failed") ? HttpStatusCode.Conflict : HttpStatusCode.Accepted); From fc60b1325aa5dcd904c0c32fb85e29d7dba36d6b Mon Sep 17 00:00:00 2001 From: sophiatev <38052607+sophiatev@users.noreply.github.com> Date: Fri, 27 Feb 2026 11:40:49 -0800 Subject: [PATCH 34/43] Update test/e2e/Apps/BasicPowerShell/LargeOutputOrchestrator/run.ps1 Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- test/e2e/Apps/BasicPowerShell/LargeOutputOrchestrator/run.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/Apps/BasicPowerShell/LargeOutputOrchestrator/run.ps1 b/test/e2e/Apps/BasicPowerShell/LargeOutputOrchestrator/run.ps1 index cc0236eb6..3aab9b64e 100644 --- a/test/e2e/Apps/BasicPowerShell/LargeOutputOrchestrator/run.ps1 +++ b/test/e2e/Apps/BasicPowerShell/LargeOutputOrchestrator/run.ps1 @@ -8,7 +8,7 @@ param($Context) $sizeInKB = [int]$Context.Input if ($sizeInKB -le 0) { - [System.ArgumentOutOfRangeException]::new("sizeInKB") + throw [System.ArgumentOutOfRangeException]::new("sizeInKB") } Write-Information "Saying hello." From 7d3ecfc33054d68768872179255e0d0a4d71948c Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Fri, 27 Feb 2026 12:07:54 -0800 Subject: [PATCH 35/43] moving the placement of the log checks in the restart tests to try to reduce the flakiness --- .../Tests/Tests/RestartOrchestrationTests.cs | 25 +++++++++++-------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/test/e2e/Tests/Tests/RestartOrchestrationTests.cs b/test/e2e/Tests/Tests/RestartOrchestrationTests.cs index baf17e078..2413fd944 100644 --- a/test/e2e/Tests/Tests/RestartOrchestrationTests.cs +++ b/test/e2e/Tests/Tests/RestartOrchestrationTests.cs @@ -38,7 +38,9 @@ public async Task RestartOrchestration_CreatedTimeAndOutputChange(bool restartWi using HttpResponseMessage response = await HttpHelpers.InvokeHttpTrigger("RestarttOrchestration_HttpStart/SimpleOrchestrator"); Assert.Equal(HttpStatusCode.Accepted, response.StatusCode); string statusQueryGetUri = await DurableHelpers.ParseStatusQueryGetUriAsync(response); - string instanceId = await DurableHelpers.ParseInstanceIdAsync(response); + string instanceId = await DurableHelpers.ParseInstanceIdAsync(response); + + await DurableHelpers.WaitForOrchestrationStateAsync(statusQueryGetUri, "Completed", 30); // Verify that the ClientOperationReceived log was emitted with a FunctionInvocationId ClientOperationLogHelpers.AssertClientOperationLogExists( @@ -47,7 +49,6 @@ public async Task RestartOrchestration_CreatedTimeAndOutputChange(bool restartWi instanceId, this.fixture.functionLanguageLocalizer.GetLanguageType()); - await DurableHelpers.WaitForOrchestrationStateAsync(statusQueryGetUri, "Completed", 30); var orchestrationDetails = await DurableHelpers.GetRunningOrchestrationDetailsAsync(statusQueryGetUri); string output1 = orchestrationDetails.Output; DateTime createdTime1 = orchestrationDetails.CreatedTime; @@ -67,7 +68,9 @@ public async Task RestartOrchestration_CreatedTimeAndOutputChange(bool restartWi "RestartOrchestration_HttpRestart", jsonBody, "application/json"); Assert.Equal(HttpStatusCode.Accepted, restartResponse.StatusCode); string restartStatusQueryGetUri = await DurableHelpers.ParseStatusQueryGetUriAsync(restartResponse); - string restartInstanceId = await DurableHelpers.ParseInstanceIdAsync(restartResponse); + string restartInstanceId = await DurableHelpers.ParseInstanceIdAsync(restartResponse); + + await DurableHelpers.WaitForOrchestrationStateAsync(restartStatusQueryGetUri, "Completed", 30); // Verify that the ClientOperationReceived log was emitted with a FunctionInvocationId ClientOperationLogHelpers.AssertClientOperationLogExists( @@ -76,7 +79,6 @@ public async Task RestartOrchestration_CreatedTimeAndOutputChange(bool restartWi instanceId, this.fixture.functionLanguageLocalizer.GetLanguageType()); - await DurableHelpers.WaitForOrchestrationStateAsync(restartStatusQueryGetUri, "Completed", 30); var restartOrchestrationDetails = await DurableHelpers.GetRunningOrchestrationDetailsAsync(restartStatusQueryGetUri); string output2 = restartOrchestrationDetails.Output; DateTime createdTime2 = restartOrchestrationDetails.CreatedTime; @@ -139,7 +141,7 @@ public async Task RestartOrchestration_NonExistentInstanceId_ShouldReturnNotFoun [Trait("Java", "Skip")] // RestartAsync not yet implemented in Java [Trait("Python", "Skip")] // RestartAsync not supported in Python [Trait("Node", "Skip")] // RestartAsync not supported in Node - public async Task RestartOrchestration_NotCompletedOrchestrationWithRestartFalse_ShouldSucceed() + public async Task RestartOrchestration_NotCompletedOrchestrationWithRestartWithNewInstanceIdFalse_ShouldSucceed() { // Start a long-running orchestration using HttpResponseMessage response = await HttpHelpers.InvokeHttpTrigger("RestarttOrchestration_HttpStart/LongOrchestrator"); @@ -148,7 +150,10 @@ public async Task RestartOrchestration_NotCompletedOrchestrationWithRestartFalse string statusQueryGetUri = await DurableHelpers.ParseStatusQueryGetUriAsync(response); DurableHelpers.OrchestrationStatusDetails orchestrationDetails = await DurableHelpers.GetRunningOrchestrationDetailsAsync(statusQueryGetUri); - DateTime createdTime1 = orchestrationDetails.CreatedTime; + DateTime createdTime1 = orchestrationDetails.CreatedTime; + + // Wait for the orchestration to be running + await DurableHelpers.WaitForOrchestrationStateAsync(statusQueryGetUri, "Running", 30); // Verify that the ClientOperationReceived log was emitted with a FunctionInvocationId ClientOperationLogHelpers.AssertClientOperationLogExists( @@ -157,9 +162,6 @@ DurableHelpers.OrchestrationStatusDetails orchestrationDetails instanceId, this.fixture.functionLanguageLocalizer.GetLanguageType()); - // Wait for the orchestration to be running - await DurableHelpers.WaitForOrchestrationStateAsync(statusQueryGetUri, "Running", 30); - // Try to restart the running orchestration with restartWithNewInstanceId = false var restartPayload = new { @@ -175,7 +177,9 @@ DurableHelpers.OrchestrationStatusDetails orchestrationDetails Assert.Equal(HttpStatusCode.Accepted, restartResponse.StatusCode); string restartStatusQueryGetUri = await DurableHelpers.ParseStatusQueryGetUriAsync(restartResponse); - string restartInstanceId = await DurableHelpers.ParseInstanceIdAsync(restartResponse); + string restartInstanceId = await DurableHelpers.ParseInstanceIdAsync(restartResponse); + + await DurableHelpers.WaitForOrchestrationStateAsync(restartStatusQueryGetUri, "Running", 30); // Verify that the ClientOperationReceived log was emitted with a FunctionInvocationId ClientOperationLogHelpers.AssertClientOperationLogExists( @@ -184,7 +188,6 @@ DurableHelpers.OrchestrationStatusDetails orchestrationDetails instanceId, this.fixture.functionLanguageLocalizer.GetLanguageType()); - await DurableHelpers.WaitForOrchestrationStateAsync(restartStatusQueryGetUri, "Running", 30); DurableHelpers.OrchestrationStatusDetails restartOrchestrationDetails = await DurableHelpers.GetRunningOrchestrationDetailsAsync(restartStatusQueryGetUri); DateTime createdTime2 = restartOrchestrationDetails.CreatedTime; From 371bc4800b208a02817a58fea53de6033459011a Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Fri, 27 Feb 2026 13:14:36 -0800 Subject: [PATCH 36/43] added conditional skip on the Suspended status for MSSQL since it doesnt support terminating suspended orchestrations --- test/e2e/Tests/Tests/DedupeStatusesTests.cs | 60 ++++++++++++--------- 1 file changed, 36 insertions(+), 24 deletions(-) diff --git a/test/e2e/Tests/Tests/DedupeStatusesTests.cs b/test/e2e/Tests/Tests/DedupeStatusesTests.cs index a02bca9b0..8cf374443 100644 --- a/test/e2e/Tests/Tests/DedupeStatusesTests.cs +++ b/test/e2e/Tests/Tests/DedupeStatusesTests.cs @@ -81,16 +81,22 @@ public async Task CanStartOrchestration_WithSameId_ForAllStatuses_ForEmptyDedupe terminateResponse.Dispose(); // Suspended - string suspendedInstanceId = Guid.NewGuid().ToString(); - using HttpResponseMessage startSuspendedResponseFirstAttempt = await StartAndWaitForState( - "LongRunningOrchestrator", suspendedInstanceId, "Running"); - await SuspendAndWaitForState(suspendedInstanceId, startSuspendedResponseFirstAttempt); - using HttpResponseMessage startSuspendedResponseSecondAttempt = await StartAndWaitForState( - "LongRunningOrchestrator", suspendedInstanceId, "Running"); - // Clean-up - terminateResponse = await HttpHelpers.InvokeHttpTrigger("TerminateInstance", $"?instanceId={suspendedInstanceId}"); - Assert.Equal(HttpStatusCode.OK, terminateResponse.StatusCode); - terminateResponse.Dispose(); + // Bug: https://github.com/microsoft/durabletask-mssql/issues/300 + // Since it is not possible to terminate a suspended orchestration in MSSQL, the start orchestration request + // will timeout waiting for the existing orchestration to terminate before creating the new one + if (this.fixture.GetDurabilityProvider() != FunctionAppFixture.ConfiguredDurabilityProviderType.MSSQL) + { + string suspendedInstanceId = Guid.NewGuid().ToString(); + using HttpResponseMessage startSuspendedResponseFirstAttempt = await StartAndWaitForState( + "LongRunningOrchestrator", suspendedInstanceId, "Running"); + await SuspendAndWaitForState(suspendedInstanceId, startSuspendedResponseFirstAttempt); + using HttpResponseMessage startSuspendedResponseSecondAttempt = await StartAndWaitForState( + "LongRunningOrchestrator", suspendedInstanceId, "Running"); + // Clean-up + terminateResponse = await HttpHelpers.InvokeHttpTrigger("TerminateInstance", $"?instanceId={suspendedInstanceId}"); + Assert.Equal(HttpStatusCode.OK, terminateResponse.StatusCode); + terminateResponse.Dispose(); + } } [Theory] @@ -174,20 +180,26 @@ public async Task StartOrchestration_WithSameId_FailsIfExistingStatus_InDedupeSt terminateResponse.Dispose(); // Suspended - string suspendedInstanceId = Guid.NewGuid().ToString(); - using HttpResponseMessage startSuspendedResponseFirstAttempt = await StartAndWaitForStateWithDedupeStatuses( - "LongRunningOrchestrator", suspendedInstanceId, "Running", dedupeStatuses); - await SuspendAndWaitForState(suspendedInstanceId, startSuspendedResponseFirstAttempt); - using HttpResponseMessage startSuspendedResponseSecondAttempt = await StartAndWaitForStateWithDedupeStatuses( - "LongRunningOrchestrator", - suspendedInstanceId, - "Running", - dedupeStatuses, - expectedCode: dedupeStatuses.Contains("Suspended") ? HttpStatusCode.Conflict : HttpStatusCode.Accepted); - // Clean-up - terminateResponse = await HttpHelpers.InvokeHttpTrigger("TerminateInstance", $"?instanceId={suspendedInstanceId}"); - Assert.Equal(HttpStatusCode.OK, terminateResponse.StatusCode); - terminateResponse.Dispose(); + // Bug: https://github.com/microsoft/durabletask-mssql/issues/300 + // Since it is not possible to terminate a suspended orchestration in MSSQL, the start orchestration request + // will timeout waiting for the existing orchestration to terminate before creating the new one + if (this.fixture.GetDurabilityProvider() != FunctionAppFixture.ConfiguredDurabilityProviderType.MSSQL) + { + string suspendedInstanceId = Guid.NewGuid().ToString(); + using HttpResponseMessage startSuspendedResponseFirstAttempt = await StartAndWaitForStateWithDedupeStatuses( + "LongRunningOrchestrator", suspendedInstanceId, "Running", dedupeStatuses); + await SuspendAndWaitForState(suspendedInstanceId, startSuspendedResponseFirstAttempt); + using HttpResponseMessage startSuspendedResponseSecondAttempt = await StartAndWaitForStateWithDedupeStatuses( + "LongRunningOrchestrator", + suspendedInstanceId, + "Running", + dedupeStatuses, + expectedCode: dedupeStatuses.Contains("Suspended") ? HttpStatusCode.Conflict : HttpStatusCode.Accepted); + // Clean-up + terminateResponse = await HttpHelpers.InvokeHttpTrigger("TerminateInstance", $"?instanceId={suspendedInstanceId}"); + Assert.Equal(HttpStatusCode.OK, terminateResponse.StatusCode); + terminateResponse.Dispose(); + } } [Theory] From ef4f29845211d65c0c21c68e5baa909938f3c800 Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Fri, 27 Feb 2026 13:58:24 -0800 Subject: [PATCH 37/43] changing the logic in the restart test to wait for the restart to complete to remove flakiness --- .../Tests/Tests/RestartOrchestrationTests.cs | 26 +++++++++++-------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/test/e2e/Tests/Tests/RestartOrchestrationTests.cs b/test/e2e/Tests/Tests/RestartOrchestrationTests.cs index 2413fd944..11dee957c 100644 --- a/test/e2e/Tests/Tests/RestartOrchestrationTests.cs +++ b/test/e2e/Tests/Tests/RestartOrchestrationTests.cs @@ -177,9 +177,21 @@ DurableHelpers.OrchestrationStatusDetails orchestrationDetails Assert.Equal(HttpStatusCode.Accepted, restartResponse.StatusCode); string restartStatusQueryGetUri = await DurableHelpers.ParseStatusQueryGetUriAsync(restartResponse); - string restartInstanceId = await DurableHelpers.ParseInstanceIdAsync(restartResponse); - - await DurableHelpers.WaitForOrchestrationStateAsync(restartStatusQueryGetUri, "Running", 30); + string restartInstanceId = await DurableHelpers.ParseInstanceIdAsync(restartResponse); + + // Wait for the "created time" to change to verify that the orchestration was restarted + DurableHelpers.OrchestrationStatusDetails restartOrchestrationDetails + = await DurableHelpers.GetRunningOrchestrationDetailsAsync(restartStatusQueryGetUri); + DateTime createdTime2 = restartOrchestrationDetails.CreatedTime; + var waitForRestartTimeout = TimeSpan.FromSeconds(30); + using CancellationTokenSource cts = new(waitForRestartTimeout); + while (!cts.IsCancellationRequested && createdTime2 == createdTime1) + { + restartOrchestrationDetails = await DurableHelpers.GetRunningOrchestrationDetailsAsync(restartStatusQueryGetUri); + createdTime2 = restartOrchestrationDetails.CreatedTime; + } + Assert.NotEqual(createdTime1, createdTime2); + Assert.Equal(instanceId, restartInstanceId); // Verify that the ClientOperationReceived log was emitted with a FunctionInvocationId ClientOperationLogHelpers.AssertClientOperationLogExists( @@ -188,14 +200,6 @@ DurableHelpers.OrchestrationStatusDetails orchestrationDetails instanceId, this.fixture.functionLanguageLocalizer.GetLanguageType()); - DurableHelpers.OrchestrationStatusDetails restartOrchestrationDetails - = await DurableHelpers.GetRunningOrchestrationDetailsAsync(restartStatusQueryGetUri); - DateTime createdTime2 = restartOrchestrationDetails.CreatedTime; - - // Created time should be different. - Assert.NotEqual(createdTime1, createdTime2); - Assert.Equal(instanceId, restartInstanceId); - // Clean up: terminate the long-running orchestration using HttpResponseMessage terminateResponse = await HttpHelpers.InvokeHttpTrigger("TerminateInstance", $"?instanceId={instanceId}"); Assert.Equal(HttpStatusCode.OK, terminateResponse.StatusCode); From da873e546521e983fa7052f18226eb80158af1fe Mon Sep 17 00:00:00 2001 From: sophiatev <38052607+sophiatev@users.noreply.github.com> Date: Fri, 27 Feb 2026 13:58:55 -0800 Subject: [PATCH 38/43] Update test/e2e/Apps/BasicNode/src/functions/LargeOutputOrchestrator.ts Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .../e2e/Apps/BasicNode/src/functions/LargeOutputOrchestrator.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/Apps/BasicNode/src/functions/LargeOutputOrchestrator.ts b/test/e2e/Apps/BasicNode/src/functions/LargeOutputOrchestrator.ts index ddbbefe58..4b6fd4a72 100644 --- a/test/e2e/Apps/BasicNode/src/functions/LargeOutputOrchestrator.ts +++ b/test/e2e/Apps/BasicNode/src/functions/LargeOutputOrchestrator.ts @@ -13,7 +13,7 @@ function generateLargeString(sizeInKB: number): string { // Orchestration const LargeOutputOrchestrator: OrchestrationHandler = function* (context: OrchestrationContext) { const sizeInKB = context.df.getInput(); - if (sizeInKB <= 0) { + if (sizeInKB == null || sizeInKB <= 0) { throw new Error('sizeInKB must be a positive integer.'); } context.log('Saying hello.'); From e6304396e103dd794c4006329a5928a7d2ae81a4 Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Fri, 27 Feb 2026 15:20:11 -0800 Subject: [PATCH 39/43] more changes to try to fix the flakiness in the tests --- test/e2e/Tests/Tests/DedupeStatusesTests.cs | 2 +- test/e2e/Tests/Tests/RestartOrchestrationTests.cs | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/test/e2e/Tests/Tests/DedupeStatusesTests.cs b/test/e2e/Tests/Tests/DedupeStatusesTests.cs index 8cf374443..280728d4d 100644 --- a/test/e2e/Tests/Tests/DedupeStatusesTests.cs +++ b/test/e2e/Tests/Tests/DedupeStatusesTests.cs @@ -62,7 +62,7 @@ public async Task CanStartOrchestration_WithSameId_ForAllStatuses_ForEmptyDedupe || this.fixture.functionLanguageLocalizer.GetLanguageType() == LanguageType.Java) { string pendingInstanceId = Guid.NewGuid().ToString(); - DateTime scheduledStartTime = DateTime.UtcNow.AddMinutes(2); + DateTime scheduledStartTime = DateTime.UtcNow.AddMinutes(10); using HttpResponseMessage startPendingResponseFirstAttempt = await StartAndWaitForState( "HelloCities", pendingInstanceId, "Pending", scheduledStartTime: scheduledStartTime); using HttpResponseMessage startPendingResponseSecondAttempt = await StartAndWaitForState( diff --git a/test/e2e/Tests/Tests/RestartOrchestrationTests.cs b/test/e2e/Tests/Tests/RestartOrchestrationTests.cs index 11dee957c..dabdbd353 100644 --- a/test/e2e/Tests/Tests/RestartOrchestrationTests.cs +++ b/test/e2e/Tests/Tests/RestartOrchestrationTests.cs @@ -171,7 +171,10 @@ DurableHelpers.OrchestrationStatusDetails orchestrationDetails string jsonBody = JsonSerializer.Serialize(restartPayload); - // Restart the orchestrator with the same instance id) + // Restart the orchestrator with the same instance id + // This is necessary to ensure that the created times for the two orchestration instances are different. + // The created time returned by the orchestration status API has a resolution only up to seconds, not milliseconds. + Thread.Sleep(TimeSpan.FromSeconds(1)); using HttpResponseMessage restartResponse = await HttpHelpers.InvokeHttpTriggerWithBody( "RestartOrchestration_HttpRestart", jsonBody, "application/json"); Assert.Equal(HttpStatusCode.Accepted, restartResponse.StatusCode); From 5d54f01381a26cc152a8b9a6c1d4635a822d814c Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Fri, 27 Feb 2026 15:20:43 -0800 Subject: [PATCH 40/43] missed a pending case in DedupeStatusesTests --- test/e2e/Tests/Tests/DedupeStatusesTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/Tests/Tests/DedupeStatusesTests.cs b/test/e2e/Tests/Tests/DedupeStatusesTests.cs index 280728d4d..e02cda8f8 100644 --- a/test/e2e/Tests/Tests/DedupeStatusesTests.cs +++ b/test/e2e/Tests/Tests/DedupeStatusesTests.cs @@ -154,7 +154,7 @@ public async Task StartOrchestration_WithSameId_FailsIfExistingStatus_InDedupeSt // Pending string pendingInstanceId = Guid.NewGuid().ToString(); - DateTime scheduledStartTime = DateTime.UtcNow.AddMinutes(2); + DateTime scheduledStartTime = DateTime.UtcNow.AddMinutes(10); using HttpResponseMessage startPendingResponseFirstAttempt = await StartAndWaitForStateWithDedupeStatuses( "HelloCities", pendingInstanceId, "Pending", dedupeStatuses, scheduledStartTime: scheduledStartTime); using HttpResponseMessage startPendingResponseSecondAttempt = await StartAndWaitForStateWithDedupeStatuses( From d25c05c35db17a5021844f47b830d7c25424f85b Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Sat, 28 Feb 2026 13:31:17 -0800 Subject: [PATCH 41/43] fixing the ScheduledStartTime typo in the tests --- test/e2e/Tests/Tests/DedupeStatusesTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/Tests/Tests/DedupeStatusesTests.cs b/test/e2e/Tests/Tests/DedupeStatusesTests.cs index e02cda8f8..6270c157c 100644 --- a/test/e2e/Tests/Tests/DedupeStatusesTests.cs +++ b/test/e2e/Tests/Tests/DedupeStatusesTests.cs @@ -229,7 +229,7 @@ private async Task StartAndWaitForState( if (scheduledStartTime is not null) { - queryString += $"&scheduledStartTime={scheduledStartTime:o}"; + queryString += $"&ScheduledStartTime={scheduledStartTime:o}"; functionName = "HelloCities_HttpStart_Scheduled"; } From b9a7453ba8b550445805e4522ec3fd2a87968b78 Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Mon, 2 Mar 2026 11:36:55 -0800 Subject: [PATCH 42/43] updated the DTS and MSSQL dependencies which unblocked certain test cases --- Directory.Packages.props | 4 +- test/e2e/Tests/Tests/DedupeStatusesTests.cs | 60 ++++++++----------- .../Tests/GetOrchestrationHistoryTests.cs | 50 +++++++--------- 3 files changed, 48 insertions(+), 66 deletions(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index d5856c25e..e94c92252 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -58,8 +58,8 @@ - - + + diff --git a/test/e2e/Tests/Tests/DedupeStatusesTests.cs b/test/e2e/Tests/Tests/DedupeStatusesTests.cs index 6270c157c..42400b9ba 100644 --- a/test/e2e/Tests/Tests/DedupeStatusesTests.cs +++ b/test/e2e/Tests/Tests/DedupeStatusesTests.cs @@ -81,22 +81,16 @@ public async Task CanStartOrchestration_WithSameId_ForAllStatuses_ForEmptyDedupe terminateResponse.Dispose(); // Suspended - // Bug: https://github.com/microsoft/durabletask-mssql/issues/300 - // Since it is not possible to terminate a suspended orchestration in MSSQL, the start orchestration request - // will timeout waiting for the existing orchestration to terminate before creating the new one - if (this.fixture.GetDurabilityProvider() != FunctionAppFixture.ConfiguredDurabilityProviderType.MSSQL) - { - string suspendedInstanceId = Guid.NewGuid().ToString(); - using HttpResponseMessage startSuspendedResponseFirstAttempt = await StartAndWaitForState( - "LongRunningOrchestrator", suspendedInstanceId, "Running"); - await SuspendAndWaitForState(suspendedInstanceId, startSuspendedResponseFirstAttempt); - using HttpResponseMessage startSuspendedResponseSecondAttempt = await StartAndWaitForState( - "LongRunningOrchestrator", suspendedInstanceId, "Running"); - // Clean-up - terminateResponse = await HttpHelpers.InvokeHttpTrigger("TerminateInstance", $"?instanceId={suspendedInstanceId}"); - Assert.Equal(HttpStatusCode.OK, terminateResponse.StatusCode); - terminateResponse.Dispose(); - } + string suspendedInstanceId = Guid.NewGuid().ToString(); + using HttpResponseMessage startSuspendedResponseFirstAttempt = await StartAndWaitForState( + "LongRunningOrchestrator", suspendedInstanceId, "Running"); + await SuspendAndWaitForState(suspendedInstanceId, startSuspendedResponseFirstAttempt); + using HttpResponseMessage startSuspendedResponseSecondAttempt = await StartAndWaitForState( + "LongRunningOrchestrator", suspendedInstanceId, "Running"); + // Clean-up + terminateResponse = await HttpHelpers.InvokeHttpTrigger("TerminateInstance", $"?instanceId={suspendedInstanceId}"); + Assert.Equal(HttpStatusCode.OK, terminateResponse.StatusCode); + terminateResponse.Dispose(); } [Theory] @@ -180,26 +174,20 @@ public async Task StartOrchestration_WithSameId_FailsIfExistingStatus_InDedupeSt terminateResponse.Dispose(); // Suspended - // Bug: https://github.com/microsoft/durabletask-mssql/issues/300 - // Since it is not possible to terminate a suspended orchestration in MSSQL, the start orchestration request - // will timeout waiting for the existing orchestration to terminate before creating the new one - if (this.fixture.GetDurabilityProvider() != FunctionAppFixture.ConfiguredDurabilityProviderType.MSSQL) - { - string suspendedInstanceId = Guid.NewGuid().ToString(); - using HttpResponseMessage startSuspendedResponseFirstAttempt = await StartAndWaitForStateWithDedupeStatuses( - "LongRunningOrchestrator", suspendedInstanceId, "Running", dedupeStatuses); - await SuspendAndWaitForState(suspendedInstanceId, startSuspendedResponseFirstAttempt); - using HttpResponseMessage startSuspendedResponseSecondAttempt = await StartAndWaitForStateWithDedupeStatuses( - "LongRunningOrchestrator", - suspendedInstanceId, - "Running", - dedupeStatuses, - expectedCode: dedupeStatuses.Contains("Suspended") ? HttpStatusCode.Conflict : HttpStatusCode.Accepted); - // Clean-up - terminateResponse = await HttpHelpers.InvokeHttpTrigger("TerminateInstance", $"?instanceId={suspendedInstanceId}"); - Assert.Equal(HttpStatusCode.OK, terminateResponse.StatusCode); - terminateResponse.Dispose(); - } + string suspendedInstanceId = Guid.NewGuid().ToString(); + using HttpResponseMessage startSuspendedResponseFirstAttempt = await StartAndWaitForStateWithDedupeStatuses( + "LongRunningOrchestrator", suspendedInstanceId, "Running", dedupeStatuses); + await SuspendAndWaitForState(suspendedInstanceId, startSuspendedResponseFirstAttempt); + using HttpResponseMessage startSuspendedResponseSecondAttempt = await StartAndWaitForStateWithDedupeStatuses( + "LongRunningOrchestrator", + suspendedInstanceId, + "Running", + dedupeStatuses, + expectedCode: dedupeStatuses.Contains("Suspended") ? HttpStatusCode.Conflict : HttpStatusCode.Accepted); + // Clean-up + terminateResponse = await HttpHelpers.InvokeHttpTrigger("TerminateInstance", $"?instanceId={suspendedInstanceId}"); + Assert.Equal(HttpStatusCode.OK, terminateResponse.StatusCode); + terminateResponse.Dispose(); } [Theory] diff --git a/test/e2e/Tests/Tests/GetOrchestrationHistoryTests.cs b/test/e2e/Tests/Tests/GetOrchestrationHistoryTests.cs index c57a85b56..9af26196d 100644 --- a/test/e2e/Tests/Tests/GetOrchestrationHistoryTests.cs +++ b/test/e2e/Tests/Tests/GetOrchestrationHistoryTests.cs @@ -44,8 +44,6 @@ public GetOrchestrationHistoryTests(FunctionAppFixture fixture, ITestOutputHelpe public async Task GetOrchestrationHistory_FailedOrchestration() { bool isNotMSSQL = this.fixture.GetDurabilityProvider() != FunctionAppFixture.ConfiguredDurabilityProviderType.MSSQL; - // The other backends currently do not serialize tags when sending the history, or the failure details of an ExecutionCompletedEvent - bool checkTagsAndFailureDetails = this.fixture.GetDurabilityProvider() == FunctionAppFixture.ConfiguredDurabilityProviderType.AzureStorage; string subOrchestrationInstanceId = Guid.NewGuid().ToString(); using HttpResponseMessage response = await HttpHelpers.InvokeHttpTrigger( @@ -79,7 +77,8 @@ public async Task GetOrchestrationHistory_FailedOrchestration() Assert.Equal("ParentOrchestration", parentExecutionStartedEvent.Name); Assert.Equal(new ComplexInput("fail", subOrchestrationInstanceId, OutputSize, isNotMSSQL, this.tags), JsonConvert.DeserializeObject(parentExecutionStartedEvent.Input)); - if (checkTagsAndFailureDetails) + // MSSQL does not include tags in history events + if (isNotMSSQL) { Assert.NotNull(parentExecutionStartedEvent.Tags); Assert.Contains(TagsKey, parentExecutionStartedEvent.Tags.Keys); @@ -114,17 +113,14 @@ public async Task GetOrchestrationHistory_FailedOrchestration() Assert.Equal("System.Exception", subOrchestrationFailureDetails.InnerFailure.ErrorType); Assert.Equal("Failure!", subOrchestrationFailureDetails.InnerFailure.ErrorMessage); - if (checkTagsAndFailureDetails) - { - Assert.NotNull(parentFailureDetails); - Assert.Equal("Microsoft.DurableTask.TaskFailedException", parentFailureDetails.ErrorType); - Assert.NotNull(parentFailureDetails.InnerFailure); - Assert.Equal("Microsoft.DurableTask.TaskFailedException", parentFailureDetails.InnerFailure.ErrorType); - Assert.Equal(subOrchestrationFailureDetails.ErrorMessage, parentFailureDetails.InnerFailure.ErrorMessage); - // Finally, the doubly nested inner failure of the execution completed event will correspond to the Activity failing - Assert.NotNull(parentFailureDetails.InnerFailure.InnerFailure); - Assert.Equal("Failure!", parentFailureDetails.InnerFailure.InnerFailure.ErrorMessage); - } + Assert.NotNull(parentFailureDetails); + Assert.Equal("Microsoft.DurableTask.TaskFailedException", parentFailureDetails.ErrorType); + Assert.NotNull(parentFailureDetails.InnerFailure); + Assert.Equal("Microsoft.DurableTask.TaskFailedException", parentFailureDetails.InnerFailure.ErrorType); + Assert.Equal(subOrchestrationFailureDetails.ErrorMessage, parentFailureDetails.InnerFailure.ErrorMessage); + // Finally, the doubly nested inner failure of the execution completed event will correspond to the Activity failing + Assert.NotNull(parentFailureDetails.InnerFailure.InnerFailure); + Assert.Equal("Failure!", parentFailureDetails.InnerFailure.InnerFailure.ErrorMessage); using HttpResponseMessage getSubOrchestrationHistoryResponse = await HttpHelpers.InvokeHttpTrigger("GetInstanceHistory", $"?instanceId={subOrchestrationInstanceId}"); Assert.Equal(HttpStatusCode.OK, getSubOrchestrationHistoryResponse.StatusCode); @@ -156,7 +152,8 @@ public async Task GetOrchestrationHistory_FailedOrchestration() Assert.Equal("ParentOrchestration", subOrchestrationExecutionStartedEvent.ParentInstance.Name); Assert.Equal(parentExecutionStartedEvent.OrchestrationInstance.ExecutionId, subOrchestrationExecutionStartedEvent.ParentInstance.OrchestrationInstance.ExecutionId); } - if (checkTagsAndFailureDetails) + // MSSQL does not include tags in history events + if (isNotMSSQL) { Assert.NotNull(subOrchestrationExecutionStartedEvent.Tags); Assert.Contains(TagsKey, subOrchestrationExecutionStartedEvent.Tags.Keys); @@ -165,7 +162,8 @@ public async Task GetOrchestrationHistory_FailedOrchestration() Assert.Equal(EventType.TaskScheduled, subOrchestrationHistoryEvents[2].EventType); var taskScheduledEvent = (TaskScheduledEvent)subOrchestrationHistoryEvents[2]; Assert.Equal("ThrowExceptionActivity", taskScheduledEvent.Name); - if (checkTagsAndFailureDetails) + // MSSQL does not include tags in history events + if (isNotMSSQL) { Assert.NotNull(taskScheduledEvent.Tags); Assert.Contains(TagsKey, taskScheduledEvent.Tags.Keys); @@ -189,15 +187,12 @@ public async Task GetOrchestrationHistory_FailedOrchestration() Assert.Equal("System.Exception", taskFailureDetails.ErrorType); Assert.Equal("Failure!", taskFailureDetails.ErrorMessage); - if (checkTagsAndFailureDetails) - { - Assert.NotNull(subOrchestrationFailureDetails); - Assert.Equal("Microsoft.DurableTask.TaskFailedException", subOrchestrationFailureDetails.ErrorType); - Assert.NotNull(subOrchestrationFailureDetails.InnerFailure); - // The inner failure for the suborchestration failed event will be the actual exception thrown by the Activity - Assert.Equal(taskFailureDetails.ErrorType, subOrchestrationFailureDetails.InnerFailure.ErrorType); - Assert.Equal(taskFailureDetails.ErrorMessage, subOrchestrationFailureDetails.InnerFailure.ErrorMessage); - } + Assert.NotNull(subOrchestrationFailureDetails); + Assert.Equal("Microsoft.DurableTask.TaskFailedException", subOrchestrationFailureDetails.ErrorType); + Assert.NotNull(subOrchestrationFailureDetails.InnerFailure); + // The inner failure for the suborchestration failed event will be the actual exception thrown by the Activity + Assert.Equal(taskFailureDetails.ErrorType, subOrchestrationFailureDetails.InnerFailure.ErrorType); + Assert.Equal(taskFailureDetails.ErrorMessage, subOrchestrationFailureDetails.InnerFailure.ErrorMessage); // Verify that the ClientOperationReceived logs were emitted with a FunctionInvocationId ClientOperationLogHelpers.AssertClientOperationLogExists( @@ -222,8 +217,6 @@ public async Task GetOrchestrationHistory_FailedOrchestration() public async Task GetOrchestrationHistory_LargeHistory() { bool isNotMSSQL = this.fixture.GetDurabilityProvider() != FunctionAppFixture.ConfiguredDurabilityProviderType.MSSQL; - // The other backends currently do not serialize tags when sending the history, or the failure details of an ExecutionCompletedEvent - bool checkTagsAndFailureDetails = this.fixture.GetDurabilityProvider() == FunctionAppFixture.ConfiguredDurabilityProviderType.AzureStorage; string subOrchestrationInstanceId = Guid.NewGuid().ToString(); using HttpResponseMessage response = await HttpHelpers.InvokeHttpTrigger( @@ -257,7 +250,8 @@ public async Task GetOrchestrationHistory_LargeHistory() Assert.Equal("ParentOrchestration", parentExecutionStartedEvent.Name); Assert.Equal(new ComplexInput("succeed", subOrchestrationInstanceId, OutputSize, isNotMSSQL, this.tags), JsonConvert.DeserializeObject(parentExecutionStartedEvent.Input)); - if (checkTagsAndFailureDetails) + // MSSQL does not include tags in history events + if (isNotMSSQL) { Assert.NotNull(parentExecutionStartedEvent.Tags); Assert.Contains(TagsKey, parentExecutionStartedEvent.Tags.Keys); From 0bdca818c65a2baccbdad444e65ffbfe690d61a0 Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Mon, 2 Mar 2026 14:28:48 -0800 Subject: [PATCH 43/43] updated the mssql dependency to the working package --- Directory.Packages.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index e94c92252..faa4ca470 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -59,7 +59,7 @@ - +