diff --git a/src/DurableTask.Netherite.AzureFunctions/NetheriteProvider.cs b/src/DurableTask.Netherite.AzureFunctions/NetheriteProvider.cs index db0abd8..f668bcc 100644 --- a/src/DurableTask.Netherite.AzureFunctions/NetheriteProvider.cs +++ b/src/DurableTask.Netherite.AzureFunctions/NetheriteProvider.cs @@ -50,6 +50,9 @@ public NetheriteProvider( public override string EventSourceName => "DurableTask-Netherite"; + NetheriteMetricsProvider metricsProvider; + ILoadPublisherService loadPublisher; + /// public async override Task RetrieveSerializedEntityState(EntityId entityId, JsonSerializerSettings serializerSettings) { @@ -130,6 +133,25 @@ public override bool TryGetScaleMonitor( } } + public override bool TryGetTargetScaler( + string functionId, + string functionName, + string hubName, + string connectionName, + out ITargetScaler targetScaler) + { + // Target Scaler is created per function id. And they share the same NetheriteMetricsProvider. + if ( this.metricsProvider == null) + { + this.loadPublisher ??= this.Service.GetLoadPublisher(); + this.metricsProvider = this.Service.GetNetheriteMetricsProvider(this.loadPublisher, this.Settings.EventHubsConnection); + } + + targetScaler = new NetheriteTargetScaler(functionId, this.metricsProvider, this); + + return true; + } + public class NetheriteScaleMetrics : ScaleMetrics { public byte[] Metrics { get; set; } diff --git a/src/DurableTask.Netherite.AzureFunctions/NetheriteTargetScaler.cs b/src/DurableTask.Netherite.AzureFunctions/NetheriteTargetScaler.cs new file mode 100644 index 0000000..9aab940 --- /dev/null +++ b/src/DurableTask.Netherite.AzureFunctions/NetheriteTargetScaler.cs @@ -0,0 +1,112 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#if !NETSTANDARD +#if !NETCOREAPP2_2 +namespace DurableTask.Netherite.AzureFunctions +{ + using System; + using System.Linq; + using System.Threading.Tasks; + using DurableTask.Netherite.Scaling; + using Microsoft.Azure.WebJobs.Extensions.DurableTask; + using Microsoft.Azure.WebJobs.Host.Scale; + using static DurableTask.Netherite.Scaling.ScalingMonitor; + + public class NetheriteTargetScaler : ITargetScaler + { + readonly NetheriteMetricsProvider metricsProvider; + readonly DurabilityProvider durabilityProvider; + readonly TargetScalerResult scaleResult; + + public NetheriteTargetScaler( + string functionId, + NetheriteMetricsProvider metricsProvider, + DurabilityProvider durabilityProvider) + { + this.metricsProvider = metricsProvider; + this.durabilityProvider = durabilityProvider; + this.scaleResult = new TargetScalerResult(); + this.TargetScalerDescriptor = new TargetScalerDescriptor(functionId); + } + + public TargetScalerDescriptor TargetScalerDescriptor { get; private set; } + + public async Task GetScaleResultAsync(TargetScalerContext context) + { + Metrics metrics = await this.metricsProvider.GetMetricsAsync(); + + int maxConcurrentActivities = this.durabilityProvider.MaxConcurrentTaskActivityWorkItems; + int maxConcurrentWorkItems = this.durabilityProvider.MaxConcurrentTaskOrchestrationWorkItems; + + int target; + + if (metrics.TaskHubIsIdle) + { + this.scaleResult.TargetWorkerCount = 0; // we need no workers + return this.scaleResult; + } + + target = 1; // always need at least one worker when we are not idle + + // if there is a backlog of activities, ask for enough workers to process them + int activities = metrics.LoadInformation.Where(info => info.Value.IsLoaded()).Sum(info => info.Value.Activities); + if (activities > 0) + { + int requestedWorkers = (activities + (maxConcurrentActivities - 1)) / maxConcurrentActivities; // rounded-up integer division + requestedWorkers = Math.Min(requestedWorkers, metrics.LoadInformation.Count); // cannot use more workers than partitions + target = Math.Max(target, requestedWorkers); + } + + // if there are load-challenged partitions, ask for a worker for each of them + int numberOfChallengedPartitions = metrics.LoadInformation.Values + .Count(info => info.IsLoaded() || info.WorkItems > maxConcurrentWorkItems); + target = Math.Max(target, numberOfChallengedPartitions); + + // Determine how many different workers are currently running + int current = metrics.LoadInformation.Values.Select(info => info.WorkerId).Distinct().Count(); + + if (target < current) + { + // the target is lower than our current scale. However, before + // scaling in, we check some more things to avoid + // over-aggressive scale-in that could impact performance negatively. + + int numberOfNonIdlePartitions = metrics.LoadInformation.Values.Count(info => !PartitionLoadInfo.IsLongIdle(info.LatencyTrend)); + if (current > numberOfNonIdlePartitions) + { + // if we have more workers than non-idle partitions, don't immediately go lower than + // the number of non-idle partitions. + target = Math.Max(target, numberOfNonIdlePartitions); + } + else + { + // All partitions are busy, so so we don't want to reduce the worker count unless load is very low. + // Even if all partitions are runnning efficiently, it can be hard to know whether it is wise to reduce the worker count. + // We want to avoid scaling in unnecessarily when we've reached optimal scale-out. + // But we also want to avoid the case where a constant trickle of load after a big scale-out prevents scaling back in. + // To balance these goals, we vote to scale down only by one worker at a time when we see this situation. + bool allPartitionsAreFast = !metrics.LoadInformation.Values.Any(info => + info.LatencyTrend.Length != PartitionLoadInfo.LatencyTrendLength + || info.LatencyTrend.Any(c => c == PartitionLoadInfo.MediumLatency || c == PartitionLoadInfo.HighLatency)); + + if (allPartitionsAreFast) + { + // don't go lower than 1 below current + target = Math.Max(target, current - 1); + } + else + { + // don't go lower than current + target = Math.Max(target, current); + } + } + } + + this.scaleResult.TargetWorkerCount = target; + return this.scaleResult; + } + } +} +#endif +#endif \ No newline at end of file diff --git a/src/DurableTask.Netherite/OrchestrationService/NetheriteOrchestrationService.cs b/src/DurableTask.Netherite/OrchestrationService/NetheriteOrchestrationService.cs index 5647c8c..4fc5081 100644 --- a/src/DurableTask.Netherite/OrchestrationService/NetheriteOrchestrationService.cs +++ b/src/DurableTask.Netherite/OrchestrationService/NetheriteOrchestrationService.cs @@ -228,6 +228,8 @@ public bool TryGetScalingMonitor(out ScalingMonitor monitor) new AzureBlobLoadPublisher(this.Settings.BlobStorageConnection, this.Settings.HubName, this.Settings.TaskhubParametersFilePath) : new AzureTableLoadPublisher(this.Settings.TableStorageConnection, this.Settings.LoadInformationAzureTableName, this.Settings.HubName); + NetheriteMetricsProvider netheriteMetricsProvider = this.GetNetheriteMetricsProvider(loadPublisher, this.Settings.EventHubsConnection); + monitor = new ScalingMonitor( loadPublisher, this.Settings.EventHubsConnection, @@ -235,7 +237,8 @@ public bool TryGetScalingMonitor(out ScalingMonitor monitor) this.Settings.HubName, this.TraceHelper.TraceScaleRecommendation, this.TraceHelper.TraceProgress, - this.TraceHelper.TraceError); + this.TraceHelper.TraceError, + netheriteMetricsProvider); return true; } @@ -249,6 +252,17 @@ public bool TryGetScalingMonitor(out ScalingMonitor monitor) return false; } + internal ILoadPublisherService GetLoadPublisher() + { + return string.IsNullOrEmpty(this.Settings.LoadInformationAzureTableName) ? + new AzureBlobLoadPublisher(this.Settings.BlobStorageConnection, this.Settings.HubName, this.Settings.TaskhubParametersFilePath) + : new AzureTableLoadPublisher(this.Settings.TableStorageConnection, this.Settings.LoadInformationAzureTableName, this.Settings.HubName); + } + + internal NetheriteMetricsProvider GetNetheriteMetricsProvider(ILoadPublisherService loadPublisher, ConnectionInfo eventHubsConnection) + { + return new NetheriteMetricsProvider(loadPublisher, eventHubsConnection); + } public void WatchThreads(object _) { diff --git a/src/DurableTask.Netherite/Scaling/NetheriteMetricsProvider.cs b/src/DurableTask.Netherite/Scaling/NetheriteMetricsProvider.cs new file mode 100644 index 0000000..bd052d4 --- /dev/null +++ b/src/DurableTask.Netherite/Scaling/NetheriteMetricsProvider.cs @@ -0,0 +1,107 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace DurableTask.Netherite.Scaling +{ + using System; + using System.Collections.Generic; + using System.Threading; + using System.Threading.Tasks; + using DurableTask.Netherite.EventHubsTransport; + using static DurableTask.Netherite.Scaling.ScalingMonitor; + + public class NetheriteMetricsProvider + { + readonly ILoadPublisherService loadPublisher; + readonly ConnectionInfo eventHubsConnection; + + DateTime lastMetricsQueryTime = DateTime.MinValue; + Metrics metrics = default; + + public NetheriteMetricsProvider( + ILoadPublisherService loadPublisher, + ConnectionInfo eventHubsConnection) + { + this.loadPublisher = loadPublisher; + this.eventHubsConnection = eventHubsConnection; + } + + public virtual async Task GetMetricsAsync() + { + DateTime now = DateTime.UtcNow; + + // Collect the metrics every 5 seconds to avoid excessive poling. + // If calling this method more frequently, return the cached metrics. + if ( now >= this.lastMetricsQueryTime.AddSeconds(5)) + { + var loadInformation = await this.loadPublisher.QueryAsync(CancellationToken.None).ConfigureAwait(false); + var busy = await this.TaskHubIsIdleAsync(loadInformation).ConfigureAwait(false); + + this.lastMetricsQueryTime = now; + this.metrics = new Metrics() + { + LoadInformation = loadInformation, + Busy = busy, + Timestamp = now, + }; + } + + return this.metrics; + } + + /// + /// Determines if a taskhub is busy, based on load information for the partitions and on the eventhubs queue positions + /// + /// + /// null if the hub is idle, or a string describing the current non-idle state + public async Task TaskHubIsIdleAsync(Dictionary loadInformation) + { + // first, check if any of the partitions have queued work or are scheduled to wake up + foreach (var kvp in loadInformation) + { + string busy = kvp.Value.IsBusy(); + if (!string.IsNullOrEmpty(busy)) + { + return $"P{kvp.Key:D2} {busy}"; + } + } + + // next, check if any of the entries are not current, in the sense that their input queue position + // does not match the latest queue position + + + List positions = await Netherite.EventHubsTransport.EventHubsConnections.GetQueuePositionsAsync(this.eventHubsConnection, EventHubsTransport.PartitionHub).ConfigureAwait(false); + + if (positions == null) + { + return "eventhubs is missing"; + } + + for (int i = 0; i < positions.Count; i++) + { + if (!loadInformation.TryGetValue((uint)i, out var loadInfo)) + { + return $"P{i:D2} has no load information published yet"; + } + if (positions[i] > loadInfo.InputQueuePosition) + { + return $"P{i:D2} has input queue position {loadInfo.InputQueuePosition} which is {positions[(int)i] - loadInfo.InputQueuePosition} behind latest position {positions[i]}"; + } + } + + // finally, check if we have waited long enough + foreach (var kvp in loadInformation) + { + string latencyTrend = kvp.Value.LatencyTrend; + + if (!PartitionLoadInfo.IsLongIdle(latencyTrend)) + { + return $"P{kvp.Key:D2} had some activity recently, latency trend is {latencyTrend}"; + } + } + + // we have concluded that there are no pending work items, timers, or unprocessed input queue entries + return null; + } + } +} diff --git a/src/DurableTask.Netherite/Scaling/ScalingMonitor.cs b/src/DurableTask.Netherite/Scaling/ScalingMonitor.cs index 37f14cc..4f31a41 100644 --- a/src/DurableTask.Netherite/Scaling/ScalingMonitor.cs +++ b/src/DurableTask.Netherite/Scaling/ScalingMonitor.cs @@ -22,6 +22,7 @@ public class ScalingMonitor readonly string partitionLoadTableName; readonly string taskHubName; readonly ILoadPublisherService loadPublisher; + readonly NetheriteMetricsProvider netheriteMetricsProvider; // public logging actions to enable collection of scale-monitor-related logging within the Netherite infrastructure public Action RecommendationTracer { get; } @@ -47,7 +48,8 @@ public ScalingMonitor( string taskHubName, Action recommendationTracer, Action informationTracer, - Action errorTracer) + Action errorTracer, + NetheriteMetricsProvider netheriteMetricsProvider) { this.RecommendationTracer = recommendationTracer; this.InformationTracer = informationTracer; @@ -57,6 +59,8 @@ public ScalingMonitor( this.eventHubsConnection = eventHubsConnection; this.partitionLoadTableName = partitionLoadTableName; this.taskHubName = taskHubName; + + this.netheriteMetricsProvider = netheriteMetricsProvider; } /// @@ -96,16 +100,7 @@ public struct Metrics /// The collected metrics. public async Task CollectMetrics() { - DateTime now = DateTime.UtcNow; - var loadInformation = await this.loadPublisher.QueryAsync(CancellationToken.None).ConfigureAwait(false); - var busy = await this.TaskHubIsIdleAsync(loadInformation).ConfigureAwait(false); - - return new Metrics() - { - LoadInformation = loadInformation, - Busy = busy, - Timestamp = now, - }; + return await this.netheriteMetricsProvider.GetMetricsAsync(); } /// @@ -195,60 +190,5 @@ bool isSlowPartition(PartitionLoadInfo info) return new ScaleRecommendation(ScaleAction.None, keepWorkersAlive: true, reason: $"Partition latencies are healthy"); } } - - /// - /// Determines if a taskhub is busy, based on load information for the partitions and on the eventhubs queue positions - /// - /// - /// null if the hub is idle, or a string describing the current non-idle state - public async Task TaskHubIsIdleAsync(Dictionary loadInformation) - { - // first, check if any of the partitions have queued work or are scheduled to wake up - foreach (var kvp in loadInformation) - { - string busy = kvp.Value.IsBusy(); - if (!string.IsNullOrEmpty(busy)) - { - return $"P{kvp.Key:D2} {busy}"; - } - } - - // next, check if any of the entries are not current, in the sense that their input queue position - // does not match the latest queue position - - - List positions = await Netherite.EventHubsTransport.EventHubsConnections.GetQueuePositionsAsync(this.eventHubsConnection, EventHubsTransport.PartitionHub).ConfigureAwait(false); - - if (positions == null) - { - return "eventhubs is missing"; - } - - for (int i = 0; i < positions.Count; i++) - { - if (!loadInformation.TryGetValue((uint) i, out var loadInfo)) - { - return $"P{i:D2} has no load information published yet"; - } - if (positions[i] > loadInfo.InputQueuePosition) - { - return $"P{i:D2} has input queue position {loadInfo.InputQueuePosition} which is {positions[(int)i] - loadInfo.InputQueuePosition} behind latest position {positions[i]}"; - } - } - - // finally, check if we have waited long enough - foreach (var kvp in loadInformation) - { - string latencyTrend = kvp.Value.LatencyTrend; - - if (!PartitionLoadInfo.IsLongIdle(latencyTrend)) - { - return $"P{kvp.Key:D2} had some activity recently, latency trend is {latencyTrend}"; - } - } - - // we have concluded that there are no pending work items, timers, or unprocessed input queue entries - return null; - } } } diff --git a/src/common.props b/src/common.props index 8a22e78..b0ad35e 100644 --- a/src/common.props +++ b/src/common.props @@ -3,7 +3,7 @@ 3 - 0 + 1 0 $(MajorVersion).$(MinorVersion).$(PatchVersion) @@ -11,4 +11,4 @@ .$(GITHUB_RUN_NUMBER) $(VersionPrefix)$(BuildSuffix) - \ No newline at end of file + diff --git a/test/DurableTask.Netherite.AzureFunctions.Tests/DurableTask.Netherite.AzureFunctions.Tests.csproj b/test/DurableTask.Netherite.AzureFunctions.Tests/DurableTask.Netherite.AzureFunctions.Tests.csproj index 4b18122..4428f4b 100644 --- a/test/DurableTask.Netherite.AzureFunctions.Tests/DurableTask.Netherite.AzureFunctions.Tests.csproj +++ b/test/DurableTask.Netherite.AzureFunctions.Tests/DurableTask.Netherite.AzureFunctions.Tests.csproj @@ -4,7 +4,7 @@ net6.0 false true - 8.0 + 12.0 ..\..\sign.snk @@ -21,6 +21,10 @@ + + + + diff --git a/test/DurableTask.Netherite.AzureFunctions.Tests/TargetBasedScalingTests.cs b/test/DurableTask.Netherite.AzureFunctions.Tests/TargetBasedScalingTests.cs new file mode 100644 index 0000000..2dee395 --- /dev/null +++ b/test/DurableTask.Netherite.AzureFunctions.Tests/TargetBasedScalingTests.cs @@ -0,0 +1,197 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace DurableTask.Netherite.AzureFunctions.Tests +{ + using System; + using System.Collections.Generic; + using System.Threading.Tasks; + using DurableTask.Core; + using DurableTask.Netherite.Scaling; + using DurableTask.Netherite.Tests; + using Microsoft.Azure.WebJobs.Extensions.DurableTask; + using Microsoft.Azure.WebJobs.Host.Scale; + using Microsoft.Extensions.Logging; + using Moq; + using Xunit; + using Xunit.Abstractions; + using static DurableTask.Netherite.Scaling.ScalingMonitor; + + public class TargetBasedScalingTests : IntegrationTestBase + { + readonly ILoggerFactory loggerFactory; + readonly Mock metricsProviderMock; + readonly Mock orchestrationServiceMock; + + public TargetBasedScalingTests(ITestOutputHelper output) : base(output) + { + this.loggerFactory = new LoggerFactory(); + var loggerProvider = new XunitLoggerProvider(); + this.loggerFactory.AddProvider(loggerProvider); + + this.orchestrationServiceMock = new Mock(MockBehavior.Strict); + + this.metricsProviderMock = new Mock( + MockBehavior.Strict, + null, + new ConnectionInfo()); + } + + [Theory] + [MemberData(nameof(Tests))] + public async Task TargetBasedScalingTest(TestData testData) + { + this.orchestrationServiceMock.Setup(m => m.MaxConcurrentTaskActivityWorkItems).Returns(testData.MaxA); + this.orchestrationServiceMock.Setup(m => m.MaxConcurrentTaskOrchestrationWorkItems).Returns(testData.MaxO); + + if (testData.Current > testData.LoadInfos.Count) + { + throw new ArgumentException("invalid test parameter", nameof(testData.Current)); + } + + var loadInformation = new Dictionary(); + for (int i = 0; i < testData.LoadInfos.Count; i++) + { + testData.LoadInfos[i].WorkerId = $"worker{Math.Min(i, testData.Current - 1)}"; + loadInformation.Add((uint)i, testData.LoadInfos[i]); + }; + + var testMetrics = new Metrics() + { + LoadInformation = loadInformation, + Busy = testData.Busy, + Timestamp = DateTime.UtcNow, + }; + + var durabilityProviderMock = new Mock( + MockBehavior.Strict, + "storageProviderName", + this.orchestrationServiceMock.Object, + new Mock().Object, + "connectionName"); + + this.metricsProviderMock.Setup(m => m.GetMetricsAsync()).ReturnsAsync(testMetrics); + + NetheriteTargetScaler targetScaler = new NetheriteTargetScaler( + "functionId", + this.metricsProviderMock.Object, + durabilityProviderMock.Object); + + TargetScalerResult result = await targetScaler.GetScaleResultAsync(new TargetScalerContext()); + + Assert.Equal(testData.Expected, result.TargetWorkerCount); + } + + public record TestData( + int Expected, + List LoadInfos, + string Busy = "busy", + int MaxA = 10, + int MaxO = 10, + int Current = 1) + { + } + + public static TheoryData Tests + { + get + { + var data = new TheoryData(); + + // if "Busy" is null, the task hub is idle and the expected target is zero + data.Add(new TestData(Expected: 0, [ Idle, FastForAWhile, NowSlow ], Busy: null)); + + // with backlog of activities, target is set to the number of activities divided by the max activities per worker (but capped by partition count) + data.Add(new TestData(Expected: 1, [Act_5000, Idle, Idle], MaxA: 5000)); + data.Add(new TestData(Expected: 1, [Act_4999, Idle, Idle], MaxA: 5000)); + data.Add(new TestData(Expected: 2, [Act_5001, Idle, Idle], MaxA: 5000)); + data.Add(new TestData(Expected: 10, [Act_5000, Idle, FastOnlyNow, Act_5000, Idle, Idle, Idle, Idle, Idle, Idle, Idle, Idle, Idle, Idle, Idle], MaxA: 1000)); + data.Add(new TestData(Expected: 6, [Act_5000, Idle, FastOnlyNow, Act_5000, Idle, Idle], MaxA: 1000)); + + // with load-challenged partitions, target is at least the number of challenged partitions + data.Add(new TestData(Expected: 4, [Act_5000, Act_5000, Act_5000, Act_5000, Idle, Idle], MaxA: 50000)); + data.Add(new TestData(Expected: 4, [Act_5000, Act_5000, NowSlow, NowVerySlow, FastOnlyNow, FastForAWhile], MaxA: 50000)); + data.Add(new TestData(Expected: 4, [Act_5000, Act_5000, NowSlow, NowVerySlow, WasSlow, Partial], MaxA: 50000)); + data.Add(new TestData(Expected: 4, [Act_5000, Act_5000, NowSlow, Orch_5000, FastOnlyNow, AlmostIdle], MaxA: 50000)); + data.Add(new TestData(Expected: 4, [Act_5000, Act_5000, NowSlow, NowVerySlow, Orch_5000, WasSlow], MaxA: 50000, MaxO: 50000)); + + // scale down: if current is above non-idle partitions, scale down to non-idle partitions + data.Add(new TestData(Expected: 5, [Act_5000, Act_5000, AlmostIdle, AlmostIdle, AlmostIdle, Idle], Current: 6, MaxA: 5000)); + data.Add(new TestData(Expected: 4, [Act_5000, Act_5000, Partial, Partial, Idle, Idle], Current: 6, MaxA: 5000)); + + // scale down below non-idle partitions: dont if some partitions are incomplete or show any slowness; otherwise by 1 + data.Add(new TestData(Expected: 4, [FastForAWhile, FastForAWhile, AlmostIdle, Partial], Current: 4)); + data.Add(new TestData(Expected: 4, [FastForAWhile, FastForAWhile, AlmostIdle, FastOnlyNow], Current: 4)); + data.Add(new TestData(Expected: 4, [FastForAWhile, FastForAWhile, AlmostIdle, WasSlow], Current: 4)); + data.Add(new TestData(Expected: 3, [FastForAWhile, FastForAWhile, AlmostIdle, AlmostIdle], Current: 4)); + data.Add(new TestData(Expected: 3, [FastForAWhile, FastForAWhile, AlmostIdle, FastForAWhile], Current: 4)); + + return data; + } + } + + static PartitionLoadInfo Idle => new PartitionLoadInfo() + { + LatencyTrend = "IIIII", + }; + + static PartitionLoadInfo FastOnlyNow => new PartitionLoadInfo() + { + LatencyTrend = "MMHML", + }; + + static PartitionLoadInfo FastForAWhile => new PartitionLoadInfo() + { + LatencyTrend = "LLLLL", + }; + + static PartitionLoadInfo Partial => new PartitionLoadInfo() + { + LatencyTrend = "I", + }; + + static PartitionLoadInfo NowSlow => new PartitionLoadInfo() + { + LatencyTrend = "IIIIM", + }; + + static PartitionLoadInfo NowVerySlow => new PartitionLoadInfo() + { + LatencyTrend = "IIIIH", + }; + + static PartitionLoadInfo WasSlow => new PartitionLoadInfo() + { + LatencyTrend = "MIIII", + }; + + static PartitionLoadInfo AlmostIdle => new PartitionLoadInfo() + { + LatencyTrend = "LIIII", + }; + + static PartitionLoadInfo Act_5000 => new PartitionLoadInfo() + { + Activities = 5000, + LatencyTrend = "MMMMM", + }; + + static PartitionLoadInfo Act_4999 => new PartitionLoadInfo() + { + Activities = 4999, + LatencyTrend = "MMMMM", + }; + + static PartitionLoadInfo Act_5001 => new PartitionLoadInfo() + { + Activities = 5001, + LatencyTrend = "MMMMM", + }; + + static PartitionLoadInfo Orch_5000 => new PartitionLoadInfo() + { + WorkItems = 5000, + LatencyTrend = "IIIII", + }; + } +} diff --git a/test/DurableTask.Netherite.Tests/DurableTask.Netherite.Tests.csproj b/test/DurableTask.Netherite.Tests/DurableTask.Netherite.Tests.csproj index 9c06deb..943dbac 100644 --- a/test/DurableTask.Netherite.Tests/DurableTask.Netherite.Tests.csproj +++ b/test/DurableTask.Netherite.Tests/DurableTask.Netherite.Tests.csproj @@ -10,6 +10,7 @@ + all