diff --git a/src/WebJobs.Script.WebHost/Controllers/HostController.cs b/src/WebJobs.Script.WebHost/Controllers/HostController.cs index ce0c73f6ca..8f214ef524 100644 --- a/src/WebJobs.Script.WebHost/Controllers/HostController.cs +++ b/src/WebJobs.Script.WebHost/Controllers/HostController.cs @@ -245,7 +245,7 @@ public async Task GetScaleStatus([FromBody] ScaleStatusContext co // TEMP: Once https://github.com/Azure/azure-functions-host/issues/5161 is fixed, we should take // FunctionsScaleManager as a parameter. - if (Utility.TryGetHostService(scriptHostManager, out FunctionsScaleManager scaleManager)) + if (Utility.TryGetHostService(scriptHostManager, out ScaleManager scaleManager)) { var scaleStatus = await scaleManager.GetScaleStatusAsync(context); return new ObjectResult(scaleStatus); diff --git a/src/WebJobs.Script.WebHost/WebJobs.Script.WebHost.csproj b/src/WebJobs.Script.WebHost/WebJobs.Script.WebHost.csproj index 7e901331e1..884fa92648 100644 --- a/src/WebJobs.Script.WebHost/WebJobs.Script.WebHost.csproj +++ b/src/WebJobs.Script.WebHost/WebJobs.Script.WebHost.csproj @@ -77,7 +77,7 @@ - + diff --git a/src/WebJobs.Script/Config/ScaleOptions.cs b/src/WebJobs.Script/Config/ScaleOptions.cs deleted file mode 100644 index c57b49880a..0000000000 --- a/src/WebJobs.Script/Config/ScaleOptions.cs +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the MIT License. See License.txt in the project root for license information. - -using System; -using Microsoft.Azure.WebJobs.Hosting; -using Newtonsoft.Json; -using Newtonsoft.Json.Linq; - -namespace Microsoft.Azure.WebJobs.Script -{ - /// - /// Defines configuration options for runtime scale monitoring. - /// - public class ScaleOptions : IOptionsFormatter - { - private TimeSpan _scaleMetricsMaxAge; - private TimeSpan _scaleMetricsSampleInterval; - - public ScaleOptions() - { - // At the default values, a single monitor will be generating 6 samples per minute - // so at 2 minutes that's 12 samples - // Assume a case of 100 functions in an app, each mapping to a monitor. Thats - // 1200 samples to read from storage on each scale status request. - ScaleMetricsMaxAge = TimeSpan.FromMinutes(2); - ScaleMetricsSampleInterval = TimeSpan.FromSeconds(10); - MetricsPurgeEnabled = true; - } - - // for testing, to allow us to bypass validations - internal ScaleOptions(TimeSpan metricsSampleInterval) : this() - { - _scaleMetricsSampleInterval = metricsSampleInterval; - } - - /// - /// Gets or sets a value indicating the maximum age for metrics. - /// Metrics that exceed this age will not be returned to monitors. - /// - public TimeSpan ScaleMetricsMaxAge - { - get - { - return _scaleMetricsMaxAge; - } - - set - { - if (value < TimeSpan.FromMinutes(1) || value > TimeSpan.FromMinutes(5)) - { - throw new ArgumentOutOfRangeException(nameof(ScaleMetricsMaxAge)); - } - _scaleMetricsMaxAge = value; - } - } - - /// - /// Gets or sets the sampling interval for metrics. - /// - public TimeSpan ScaleMetricsSampleInterval - { - get - { - return _scaleMetricsSampleInterval; - } - - set - { - if (value < TimeSpan.FromSeconds(1) || value > TimeSpan.FromSeconds(30)) - { - throw new ArgumentOutOfRangeException(nameof(ScaleMetricsSampleInterval)); - } - _scaleMetricsSampleInterval = value; - } - } - - /// - /// Gets or sets a value indicating whether old metrics data - /// will be auto purged. - /// - public bool MetricsPurgeEnabled { get; set; } - - public string Format() - { - var options = new JObject - { - { nameof(ScaleMetricsMaxAge), ScaleMetricsMaxAge }, - { nameof(ScaleMetricsSampleInterval), ScaleMetricsSampleInterval } - }; - - return options.ToString(Formatting.Indented); - } - } -} diff --git a/src/WebJobs.Script/Config/ScaleOptionsSetup.cs b/src/WebJobs.Script/Config/ScaleOptionsSetup.cs new file mode 100644 index 0000000000..a4c923b4d7 --- /dev/null +++ b/src/WebJobs.Script/Config/ScaleOptionsSetup.cs @@ -0,0 +1,34 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using Microsoft.Azure.WebJobs.Host.Scale; +using Microsoft.Extensions.Options; + +namespace Microsoft.Azure.WebJobs.Script.Config +{ + public class ScaleOptionsSetup : IConfigureOptions + { + private readonly IOptions _functionsHostingConfigOptions; + private readonly IEnvironment _environment; + + public ScaleOptionsSetup(IEnvironment environment, IOptions functionsHostingConfigOptions) + { + _environment = environment; + _functionsHostingConfigOptions = functionsHostingConfigOptions; + } + + public void Configure(ScaleOptions options) + { + options.IsTargetBasedScalingEnabled = _environment.IsTargetBasedScalingEnabled(); + options.IsTargetBasedScalingEnabledForTriggerFunc = IsTargetBasedScalingEnabledForTrigger; + } + + private bool IsTargetBasedScalingEnabledForTrigger(ITargetScaler targetScaler) + { + string assemblyName = targetScaler.GetType().Assembly.GetName().Name; + string flag = _functionsHostingConfigOptions.Value.GetFeature(assemblyName); + return flag == "1"; + } + } +} diff --git a/src/WebJobs.Script/Scale/FunctionsScaleManager.cs b/src/WebJobs.Script/Scale/FunctionsScaleManager.cs deleted file mode 100644 index 6cb3f081cb..0000000000 --- a/src/WebJobs.Script/Scale/FunctionsScaleManager.cs +++ /dev/null @@ -1,294 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the MIT License. See License.txt in the project root for license information. - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Reflection; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Azure.AppService.Proxy.Client; -using Microsoft.Azure.WebJobs.Host.Scale; -using Microsoft.Azure.WebJobs.Script.Config; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; - -namespace Microsoft.Azure.WebJobs.Script.Scale -{ - /// - /// Manages scale monitoring operations. - /// - public class FunctionsScaleManager - { - private readonly IScaleMonitorManager _monitorManager; - private readonly IScaleMetricsRepository _metricsRepository; - private readonly ITargetScalerManager _targetScalerManager; - private readonly IConcurrencyStatusRepository _concurrencyStatusRepository; - private readonly IOptions _functionsHostingConfigOptions; - private readonly IEnvironment _environment; - private readonly ILogger _logger; - private readonly HashSet _targetScalersInError; - - // for mock testing only - public FunctionsScaleManager() - { - } - - public FunctionsScaleManager( - IScaleMonitorManager monitorManager, - IScaleMetricsRepository metricsRepository, - ITargetScalerManager targetScalerManager, - IConcurrencyStatusRepository concurrencyStatusRepository, - IOptions functionsHostingConfigOptions, - IEnvironment environment, - ILoggerFactory loggerFactory) - { - _monitorManager = monitorManager; - _metricsRepository = metricsRepository; - _targetScalerManager = targetScalerManager; - _concurrencyStatusRepository = concurrencyStatusRepository; - _functionsHostingConfigOptions = functionsHostingConfigOptions; - _environment = environment; - _logger = loggerFactory.CreateLogger(); - _targetScalersInError = new HashSet(); - } - - /// - /// Get the current scale status (vote) by querying all active monitors for their - /// scale status. - /// - /// The context to use for the scale decision. - /// The scale vote. - public virtual async Task GetScaleStatusAsync(ScaleStatusContext context) - { - GetScalersToSample(out List scaleMonitorsToProcess, out List targetScalersToProcess); - - var scaleMonitorVotes = await GetScaleMonitorsResultAsync(context, scaleMonitorsToProcess); - var targetScalerVotes = await GetTargetScalersResultAsync(context, targetScalersToProcess); - - return new ScaleStatusResult - { - Vote = GetAggregateScaleVote(scaleMonitorVotes.Union(targetScalerVotes.Select(x => x.Vote)), context, _logger), - TargetWorkerCount = targetScalerVotes.Any() ? targetScalerVotes.Max(x => x.TargetWorkerCount) : null - }; - } - - private async Task> GetScaleMonitorsResultAsync(ScaleStatusContext context, IEnumerable scaleMonitorsToProcess) - { - List votes = new List(); - if (scaleMonitorsToProcess.Any()) - { - // get the collection of current metrics for each monitor - var monitorMetrics = await _metricsRepository.ReadMetricsAsync(scaleMonitorsToProcess); - - _logger.LogDebug($"Computing scale status (WorkerCount={context.WorkerCount})"); - _logger.LogDebug($"{monitorMetrics.Count} scale monitors to sample"); - - // for each monitor, ask it to return its scale status (vote) based on - // the metrics and context info (e.g. worker count) - foreach (var pair in monitorMetrics) - { - var monitor = pair.Key; - var metrics = pair.Value; - - try - { - // create a new context instance to avoid modifying the - // incoming context - var scaleStatusContext = new ScaleStatusContext - { - WorkerCount = context.WorkerCount, - Metrics = metrics - }; - var result = monitor.GetScaleStatus(scaleStatusContext); - - _logger.LogDebug($"Monitor '{monitor.Descriptor.Id}' voted '{result.Vote.ToString()}'"); - votes.Add(result.Vote); - } - catch (Exception exc) when (!exc.IsFatal()) - { - // if a particular monitor fails, log and continue - _logger.LogError(exc, $"Failed to query scale status for monitor '{monitor.Descriptor.Id}'."); - } - } - } - else - { - // no monitors registered - // this can happen if the host is offline - } - - return votes; - } - - private async Task> GetTargetScalersResultAsync(ScaleStatusContext context, IEnumerable targetScalersToProcess) - { - List targetScaleVotes = new List(); - - if (targetScalersToProcess.Any()) - { - _logger.LogDebug($"{targetScalersToProcess.Count()} target scalers to sample"); - HostConcurrencySnapshot snapshot = null; - try - { - snapshot = await _concurrencyStatusRepository.ReadAsync(CancellationToken.None); - } - catch (Exception exc) when (!exc.IsFatal()) - { - _logger.LogError(exc, $"Failed to read concurrency status repository"); - } - - foreach (var targetScaler in targetScalersToProcess) - { - try - { - TargetScalerContext targetScaleStatusContext = new TargetScalerContext(); - if (snapshot != null) - { - if (snapshot.FunctionSnapshots.TryGetValue(targetScaler.TargetScalerDescriptor.FunctionId, out var functionSnapshot)) - { - targetScaleStatusContext.InstanceConcurrency = functionSnapshot.Concurrency; - _logger.LogDebug($"Snapshot dynamic concurrency for target scaler '{targetScaler.TargetScalerDescriptor.FunctionId}' is '{functionSnapshot.Concurrency}'"); - } - } - TargetScalerResult result = null; - try - { - result = await targetScaler.GetScaleResultAsync(targetScaleStatusContext); - } - catch (NotSupportedException ex) - { - string targetScalerUniqueId = GetTargetScalerFunctionUniqueId(targetScaler); - _logger.LogWarning($"Unable to use target based scaling for Function '{targetScaler.TargetScalerDescriptor.FunctionId}'. Metrics monitoring will be used.", ex); - _targetScalersInError.Add(targetScalerUniqueId); - - // Adding ScaleVote.None vote - result = new TargetScalerResult - { - TargetWorkerCount = context.WorkerCount - }; - } - _logger.LogDebug($"Target worker count for '{targetScaler.TargetScalerDescriptor.FunctionId}' is '{result.TargetWorkerCount}'"); - - ScaleVote vote = ScaleVote.None; - if (context.WorkerCount > result.TargetWorkerCount) - { - vote = ScaleVote.ScaleIn; - } - else if (context.WorkerCount < result.TargetWorkerCount) - { - vote = ScaleVote.ScaleOut; - } - - targetScaleVotes.Add(new TargetScalerVote - { - TargetWorkerCount = result.TargetWorkerCount, - Vote = vote - }); - } - catch (Exception exc) when (!exc.IsFatal()) - { - // if a particular target scaler fails, log and continue - _logger.LogError(exc, $"Failed to query scale result for target scaler '{targetScaler.TargetScalerDescriptor.FunctionId}'."); - } - } - } - return targetScaleVotes; - } - - internal static ScaleVote GetAggregateScaleVote(IEnumerable votes, ScaleStatusContext context, ILogger logger) - { - ScaleVote vote = ScaleVote.None; - if (votes.Any()) - { - // aggregate all the votes into a single vote - if (votes.Any(p => p == ScaleVote.ScaleOut)) - { - // scale out if at least 1 monitor requires it - logger.LogDebug("Scaling out based on votes"); - vote = ScaleVote.ScaleOut; - } - else if (context.WorkerCount > 0 && votes.All(p => p == ScaleVote.ScaleIn)) - { - // scale in only if all monitors vote scale in - logger.LogDebug("Scaling in based on votes"); - vote = ScaleVote.ScaleIn; - } - } - else if (context.WorkerCount > 0) - { - // if no functions exist or are enabled we'll scale in - logger.LogDebug("No enabled functions or scale votes so scaling in"); - vote = ScaleVote.ScaleIn; - } - - return vote; - } - - /// - /// Returns scale monitors and target scalers we want to use based on the configuration. - /// Scaler monitor will be ignored if a target scaler is defined in the same extensions assembly and TBS is enabled. - /// - /// Scale monitor to process. - /// Target scaler to process. - public virtual void GetScalersToSample( - out List scaleMonitorsToSample, - out List targetScalersToSample) - { - var scaleMonitors = _monitorManager.GetMonitors(); - var targetScalers = _targetScalerManager.GetTargetScalers(); - - scaleMonitorsToSample = new List(); - targetScalersToSample = new List(); - - // Check if TBS enabled on app level - if (_environment.IsTargetBasedScalingEnabled()) - { - HashSet targetScalerFunctions = new HashSet(); - foreach (var scaler in targetScalers) - { - string scalerUniqueId = GetTargetScalerFunctionUniqueId(scaler); - if (!_targetScalersInError.Contains(scalerUniqueId)) - { - string assemblyName = GetAssemblyName(scaler.GetType()); - string flag = _functionsHostingConfigOptions.Value.GetFeature(assemblyName); - if (flag == "1") - { - targetScalersToSample.Add(scaler); - targetScalerFunctions.Add(scalerUniqueId); - } - } - } - - foreach (var monitor in scaleMonitors) - { - string monitorUniqueId = GetScaleMonitorFunctionUniqueId(monitor); - // Check if target based scaler exists for the function - if (!targetScalerFunctions.Contains(monitorUniqueId)) - { - scaleMonitorsToSample.Add(monitor); - } - } - } - else - { - scaleMonitorsToSample.AddRange(scaleMonitors); - } - } - - private string GetTargetScalerFunctionUniqueId(ITargetScaler scaler) - { - return $"{GetAssemblyName(scaler.GetType())}-{scaler.TargetScalerDescriptor.FunctionId}"; - } - - private string GetScaleMonitorFunctionUniqueId(IScaleMonitor monitor) - { - return $"{GetAssemblyName(monitor.GetType())}-{monitor.Descriptor.FunctionId}"; - } - - private string GetAssemblyName(Type type) - { - return type.Assembly.GetName().Name; - } - } -} diff --git a/src/WebJobs.Script/Scale/FunctionsScaleMonitorService.cs b/src/WebJobs.Script/Scale/FunctionsScaleMonitorService.cs index 12ba8c48c2..f689f1c7a6 100644 --- a/src/WebJobs.Script/Scale/FunctionsScaleMonitorService.cs +++ b/src/WebJobs.Script/Scale/FunctionsScaleMonitorService.cs @@ -1,159 +1,43 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the MIT License. See License.txt in the project root for license information. -using System; -using System.Collections.Generic; -using System.Linq; using System.Threading; using System.Threading.Tasks; using Microsoft.Azure.WebJobs.Host.Scale; using Microsoft.Azure.WebJobs.Hosting; -using Microsoft.Azure.WebJobs.Script.Config; -using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; -using Newtonsoft.Json; namespace Microsoft.Azure.WebJobs.Script.Scale { - /// - /// Service responsible for taking periodic scale metrics samples and persisting them. - /// - public class FunctionsScaleMonitorService : IHostedService, IDisposable + public class FunctionsScaleMonitorService : ScaleMonitorService { private readonly IPrimaryHostStateProvider _primaryHostStateProvider; - private readonly IScaleMetricsRepository _metricsRepository; private readonly IEnvironment _environment; - private readonly ILogger _logger; - private readonly Timer _timer; - private readonly TimeSpan _interval; - private readonly ScaleOptions _scaleOptions; - private readonly FunctionsScaleManager _functionsScaleManager; - private bool _disposed; - public FunctionsScaleMonitorService(FunctionsScaleManager functionsScaleManager, IScaleMetricsRepository metricsRepository, IPrimaryHostStateProvider primaryHostStateProvider, IEnvironment environment, ILoggerFactory loggerFactory, IOptions scaleOptions) + public FunctionsScaleMonitorService(ScaleManager scaleManager, IScaleMetricsRepository metricsRepository, ILoggerFactory loggerFactory, + IPrimaryHostStateProvider primaryHostStateProvider, IEnvironment environment, IOptions scaleOptions) : base(scaleManager, metricsRepository, scaleOptions, loggerFactory) { - _metricsRepository = metricsRepository; _primaryHostStateProvider = primaryHostStateProvider; _environment = environment; - _logger = loggerFactory.CreateLogger(); - _scaleOptions = scaleOptions.Value; - - _interval = _scaleOptions.ScaleMetricsSampleInterval; - _timer = new Timer(OnTimer, null, Timeout.Infinite, Timeout.Infinite); - _functionsScaleManager = functionsScaleManager; } - public Task StartAsync(CancellationToken cancellationToken) + public override Task StartAsync(CancellationToken cancellationToken) { if (_environment.IsRuntimeScaleMonitoringEnabled()) { - _logger.LogInformation("Runtime scale monitoring is enabled."); - - // start the timer by setting the due time - SetTimerInterval((int)_interval.TotalMilliseconds); + base.StartAsync(cancellationToken); } return Task.CompletedTask; } - public Task StopAsync(CancellationToken cancellationToken) - { - // stop the timer if it has been started - _timer.Change(Timeout.Infinite, Timeout.Infinite); - - return Task.CompletedTask; - } - - private async void OnTimer(object state) + protected override void OnTimer(object state) { if (_primaryHostStateProvider.IsPrimary) { - await TakeMetricsSamplesAsync(); - } - - SetTimerInterval((int)_interval.TotalMilliseconds); - } - - internal async Task TakeMetricsSamplesAsync() - { - try - { - _functionsScaleManager.GetScalersToSample(out List scaleMonitorsToProcess, out List targetScalersToProcess); - - if (scaleMonitorsToProcess.Any()) - { - _logger.LogDebug($"Taking metrics samples for {scaleMonitorsToProcess.Count()} monitor(s)."); - - var metricsMap = new Dictionary(); - foreach (var monitor in scaleMonitorsToProcess) - { - ScaleMetrics metrics = null; - try - { - // take a metrics sample for each monitor - metrics = await monitor.GetMetricsAsync(); - metricsMap[monitor] = metrics; - - // log the metrics json to provide visibility into monitor activity - var json = JsonConvert.SerializeObject(metrics); - _logger.LogDebug($"Scale metrics sample for monitor '{monitor.Descriptor.Id}': {json}"); - } - catch (Exception exc) when (!exc.IsFatal()) - { - // if a particular monitor fails, log and continue - _logger.LogError(exc, $"Failed to collect scale metrics sample for monitor '{monitor.Descriptor.Id}'."); - } - } - - if (metricsMap.Count > 0) - { - // persist the metrics samples - await _metricsRepository.WriteMetricsAsync(metricsMap); - } - } + base.OnTimer(state); } - catch (Exception exc) when (!exc.IsFatal()) - { - _logger.LogError(exc, "Failed to collect/persist scale metrics."); - } - } - - private void SetTimerInterval(int dueTime) - { - if (!_disposed) - { - var timer = _timer; - if (timer != null) - { - try - { - _timer.Change(dueTime, Timeout.Infinite); - } - catch (ObjectDisposedException) - { - // might race with dispose - } - } - } - } - - private void Dispose(bool disposing) - { - if (!_disposed) - { - if (disposing) - { - _timer.Dispose(); - } - - _disposed = true; - } - } - - public void Dispose() - { - Dispose(true); } } -} +} \ No newline at end of file diff --git a/src/WebJobs.Script/Scale/IScaleMetricsRepository.cs b/src/WebJobs.Script/Scale/IScaleMetricsRepository.cs deleted file mode 100644 index 3aa6c6e692..0000000000 --- a/src/WebJobs.Script/Scale/IScaleMetricsRepository.cs +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the MIT License. See License.txt in the project root for license information. - -using System.Collections.Generic; -using System.Threading.Tasks; -using Microsoft.Azure.WebJobs.Host.Scale; - -namespace Microsoft.Azure.WebJobs.Script.Scale -{ - /// - /// Interface defining methods for reading/writing scale metrics to a persistent store. - /// - public interface IScaleMetricsRepository - { - /// - /// Persist the metrics for each monitor. - /// - /// The collection of metrics for each monitor. - /// A task. - Task WriteMetricsAsync(IDictionary monitorMetrics); - - /// - /// Read the metrics. - /// - /// The current collection of monitors. - /// Map of metrics per monitor. - Task>> ReadMetricsAsync(IEnumerable monitors); - } -} diff --git a/src/WebJobs.Script/Scale/ScaleStatusResult.cs b/src/WebJobs.Script/Scale/ScaleStatusResult.cs deleted file mode 100644 index 1ccd2b8dd5..0000000000 --- a/src/WebJobs.Script/Scale/ScaleStatusResult.cs +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the MIT License. See License.txt in the project root for license information. - -using Microsoft.Azure.WebJobs.Host.Scale; - -namespace Microsoft.Azure.WebJobs.Script.Scale -{ - /// - /// Represents an aggregate scale status result returned from - /// - /// - /// We use a separate model type for this rather than reusing to decouple these contracts. This type - /// is used as the contract with ScaleController and is serialized externally. is the contract between - /// the host and binding extensions. - /// - public class ScaleStatusResult - { - public ScaleVote Vote { get; set; } - - public int? TargetWorkerCount { get; set; } - } -} diff --git a/src/WebJobs.Script/ScriptHostBuilderExtensions.cs b/src/WebJobs.Script/ScriptHostBuilderExtensions.cs index 53e5b1126b..b84cb26ea2 100644 --- a/src/WebJobs.Script/ScriptHostBuilderExtensions.cs +++ b/src/WebJobs.Script/ScriptHostBuilderExtensions.cs @@ -8,6 +8,7 @@ using Microsoft.ApplicationInsights.WindowsServer.TelemetryChannel; using Microsoft.Azure.WebJobs.Host; using Microsoft.Azure.WebJobs.Host.Executors; +using Microsoft.Azure.WebJobs.Host.Scale; using Microsoft.Azure.WebJobs.Hosting; using Microsoft.Azure.WebJobs.Logging; using Microsoft.Azure.WebJobs.Logging.ApplicationInsights; @@ -307,7 +308,7 @@ public static IHostBuilder AddScriptHostCore(this IHostBuilder builder, ScriptAp { services.TryAddEnumerable(ServiceDescriptor.Singleton()); } - services.TryAddSingleton(); + services.TryAddSingleton(); services.AddSingleton(); }); diff --git a/src/WebJobs.Script/WebJobs.Script.csproj b/src/WebJobs.Script/WebJobs.Script.csproj index e8057252ae..3fdd2ec362 100644 --- a/src/WebJobs.Script/WebJobs.Script.csproj +++ b/src/WebJobs.Script/WebJobs.Script.csproj @@ -1,4 +1,4 @@ - + Microsoft.Azure.WebJobs.Script @@ -47,7 +47,7 @@ - + diff --git a/test/WebJobs.Script.Tests/Configuration/ScaleOptionsSetupTests.cs b/test/WebJobs.Script.Tests/Configuration/ScaleOptionsSetupTests.cs new file mode 100644 index 0000000000..de6c19f3e3 --- /dev/null +++ b/test/WebJobs.Script.Tests/Configuration/ScaleOptionsSetupTests.cs @@ -0,0 +1,40 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using Microsoft.Azure.WebJobs.Host.Scale; +using Microsoft.Azure.WebJobs.Script.Config; +using Microsoft.Extensions.Options; +using Xunit; + +namespace Microsoft.Azure.WebJobs.Script.Tests.Configuration +{ + public class ScaleOptionsSetupTests + { + [Theory] + [InlineData(false)] + [InlineData(true)] + public void Configure_SetsExpectedValues(bool tbsEnabled) + { + var options = new ScaleOptions(); + IOptions hostingConfigOptions = Options.Create(new FunctionsHostingConfigOptions()); + hostingConfigOptions.Value.Features[GetType().Assembly.GetName().Name] = "1"; + + TestEnvironment testEnvironment = new TestEnvironment(); + if (tbsEnabled) + { + testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.TargetBaseScalingEnabled, "1"); + } + + var setup = new ScaleOptionsSetup(testEnvironment, hostingConfigOptions); + setup.Configure(options); + + TestTargetScaler scaler = new TestTargetScaler(); + Assert.True(options.IsTargetBasedScalingEnabled == tbsEnabled); + Assert.True(options.IsTargetBasedScalingEnabledForTriggerFunc(scaler)); + Assert.Equal(options.ScaleMetricsMaxAge, TimeSpan.FromMinutes(2)); + Assert.Equal(options.ScaleMetricsSampleInterval, TimeSpan.FromSeconds(10)); + Assert.Equal(options.MetricsPurgeEnabled, true); + } + } +} diff --git a/test/WebJobs.Script.Tests/Controllers/Admin/HostControllerTests.cs b/test/WebJobs.Script.Tests/Controllers/Admin/HostControllerTests.cs index d112dc8a6e..c550e9e04d 100644 --- a/test/WebJobs.Script.Tests/Controllers/Admin/HostControllerTests.cs +++ b/test/WebJobs.Script.Tests/Controllers/Admin/HostControllerTests.cs @@ -148,14 +148,12 @@ public async Task GetScaleStatus_RuntimeScaleModeEnabled_Succeeds() { WorkerCount = 5 }; - var scaleManagerMock = new Mock(MockBehavior.Strict); - var scaleStatusResult = new ScaleStatusResult { Vote = ScaleVote.ScaleOut, TargetWorkerCount = 2 }; - scaleManagerMock.Setup(p => p.GetScaleStatusAsync(context)).ReturnsAsync(scaleStatusResult); + var scaleManagerMock = new Mock(MockBehavior.Loose); var scriptHostManagerMock = new Mock(MockBehavior.Strict); var serviceProviderMock = scriptHostManagerMock.As(); - serviceProviderMock.Setup(p => p.GetService(typeof(FunctionsScaleManager))).Returns(scaleManagerMock.Object); + serviceProviderMock.Setup(p => p.GetService(typeof(ScaleManager))).Returns(scaleManagerMock.Object); var result = (ObjectResult)(await _hostController.GetScaleStatus(context, scriptHostManagerMock.Object)); - Assert.Same(result.Value, scaleStatusResult); + Assert.Equal(((ScaleStatus)result.Value).Vote, ScaleVote.ScaleIn); } [Fact] @@ -169,7 +167,7 @@ public async Task GetScaleStatus_FunctionsScaleManager_Null_ReturnsServiceUnavai }; var scriptHostManagerMock = new Mock(MockBehavior.Strict); var serviceProviderMock = scriptHostManagerMock.As(); - serviceProviderMock.Setup(p => p.GetService(typeof(FunctionsScaleManager))).Returns(null); + serviceProviderMock.Setup(p => p.GetService(typeof(ScaleManager))).Returns(null); var result = (StatusCodeResult)(await _hostController.GetScaleStatus(context, scriptHostManagerMock.Object)); Assert.Equal(StatusCodes.Status503ServiceUnavailable, result.StatusCode); @@ -182,10 +180,10 @@ public async Task GetScaleStatus_RuntimeScaleModeNotEnabled_ReturnsBadRequest() { WorkerCount = 5 }; - var scaleManagerMock = new Mock(MockBehavior.Strict); + var scaleManagerMock = new Mock(MockBehavior.Strict); var scriptHostManagerMock = new Mock(MockBehavior.Strict); var serviceProviderMock = scriptHostManagerMock.As(); - serviceProviderMock.Setup(p => p.GetService(typeof(FunctionsScaleManager))).Returns(scaleManagerMock.Object); + serviceProviderMock.Setup(p => p.GetService(typeof(ScaleManager))).Returns(scaleManagerMock.Object); var result = (BadRequestObjectResult)(await _hostController.GetScaleStatus(context, scriptHostManagerMock.Object)); Assert.Equal(HttpStatusCode.BadRequest, (HttpStatusCode)result.StatusCode); Assert.Equal("Runtime scale monitoring is not enabled.", result.Value); diff --git a/test/WebJobs.Script.Tests/Scale/FunctionsScaleManagerTests.cs b/test/WebJobs.Script.Tests/Scale/FunctionsScaleManagerTests.cs deleted file mode 100644 index f09c292bbc..0000000000 --- a/test/WebJobs.Script.Tests/Scale/FunctionsScaleManagerTests.cs +++ /dev/null @@ -1,387 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the MIT License. See License.txt in the project root for license information. - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Reflection; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Azure.WebJobs.Host.Scale; -using Microsoft.Azure.WebJobs.Script.Config; -using Microsoft.Azure.WebJobs.Script.Scale; -using Microsoft.Azure.WebJobs.Script.Tests.Description.DotNet; -using Microsoft.Azure.WebJobs.Script.WebHost.Filters; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; -using Microsoft.WebJobs.Script.Tests; -using Moq; -using Xunit; - -namespace Microsoft.Azure.WebJobs.Script.Tests.Scale -{ - public class FunctionsScaleManagerTests - { - private readonly FunctionsScaleManager _scaleManager; - private readonly Mock _monitorManagerMock; - private readonly Mock _metricsRepositoryMock; - private readonly Mock _targetScalerManagerMock; - private readonly Mock _concurrencyStatusRepositoryMock; - private readonly ILoggerFactory _loggerFactory; - private readonly TestLoggerProvider _loggerProvider; - private readonly List _monitors; - private readonly List _targetScalers; - private readonly TestEnvironment _environment; - private readonly ILogger _testLogger; - private IOptions _functionsHostingConfigOptions; - - public FunctionsScaleManagerTests() - { - _monitors = new List(); - _targetScalers = new List(); - _loggerProvider = new TestLoggerProvider(); - _loggerFactory = new LoggerFactory(); - _loggerFactory.AddProvider(_loggerProvider); - _testLogger = _loggerFactory.CreateLogger("Test"); - - _monitorManagerMock = new Mock(MockBehavior.Strict); - _monitorManagerMock.Setup(p => p.GetMonitors()).Returns(() => _monitors); - _metricsRepositoryMock = new Mock(MockBehavior.Strict); - _metricsRepositoryMock.Setup(x => x.ReadMetricsAsync(It.IsAny>())).ReturnsAsync(new Dictionary>()); - _targetScalerManagerMock = new Mock(MockBehavior.Strict); - _targetScalerManagerMock.Setup(p => p.GetTargetScalers()).Returns(() => _targetScalers); - _concurrencyStatusRepositoryMock = new Mock(MockBehavior.Strict); - _concurrencyStatusRepositoryMock.Setup(p => p.ReadAsync(It.IsAny())).ReturnsAsync( - new HostConcurrencySnapshot() - { - FunctionSnapshots = new Dictionary() - { - { "func1", new FunctionConcurrencySnapshot() { Concurrency = 1 } } - } - }); - - _functionsHostingConfigOptions = Options.Create(new FunctionsHostingConfigOptions()); - - _environment = new TestEnvironment(); - - _scaleManager = new FunctionsScaleManager(_monitorManagerMock.Object, _metricsRepositoryMock.Object, _targetScalerManagerMock.Object, _concurrencyStatusRepositoryMock.Object, - _functionsHostingConfigOptions, _environment, _loggerFactory); - } - - [Theory] - [InlineData(0, ScaleVote.None)] - [InlineData(1, ScaleVote.ScaleIn)] - public async Task GetScaleStatus_NoMonitors_ReturnsExpectedStatus(int workerCount, ScaleVote expected) - { - var context = new ScaleStatusContext - { - WorkerCount = workerCount - }; - var status = await _scaleManager.GetScaleStatusAsync(context); - - Assert.Equal(expected, status.Vote); - } - - [Theory] - [InlineData(true)] - [InlineData(false)] - public async Task GetScaleStatus_ReturnsExpectedResult(bool tbsEnabled) - { - var context = new ScaleStatusContext - { - WorkerCount = 3 - }; - - var monitor1 = new TestScaleMonitor("func1-test-test", "func1"); - monitor1.Status = new ScaleStatus - { - Vote = ScaleVote.ScaleIn - }; - var monitor2 = new TestScaleMonitor1(); - monitor2.Status = new ScaleStatus - { - Vote = ScaleVote.ScaleOut - }; - var monitor3 = new TestScaleMonitor1(); - monitor3.Status = new ScaleStatus - { - Vote = ScaleVote.ScaleIn - }; - - List monitors = new List - { - monitor1 - }; - if (!tbsEnabled) - { - monitors.Add(monitor2); - monitors.Add(monitor3); - } - _monitorManagerMock.Setup(p => p.GetMonitors()).Returns(monitors); - - var monitorMetrics = new Dictionary> - { - { monitor1, new List() }, - { monitor2, new List() }, - { monitor3, new List() } - }; - _metricsRepositoryMock.Setup(p => p.ReadMetricsAsync(It.IsAny>())).ReturnsAsync(monitorMetrics); - - var targetScaler1 = new TestTargetScaler() - { - Result = new TargetScalerResult() - { - TargetWorkerCount = 2 - }, - TargetScalerDescriptor = new TargetScalerDescriptor("func1") - }; - - _targetScalerManagerMock.Setup(p => p.GetTargetScalers()).Returns(new List { targetScaler1 }); - - if (tbsEnabled) - { - _environment.SetEnvironmentVariable(EnvironmentSettingNames.TargetBaseScalingEnabled, "1"); - } - - string assemblyName = Assembly.GetExecutingAssembly().GetName().Name; - _functionsHostingConfigOptions.Value.Features[assemblyName] = "1"; - - var status = await _scaleManager.GetScaleStatusAsync(context); - - var logs = _loggerProvider.GetAllLogMessages(); - if (!tbsEnabled) - { - Assert.Equal("Computing scale status (WorkerCount=3)", logs[0].FormattedMessage); - Assert.Equal("3 scale monitors to sample", logs[1].FormattedMessage); - Assert.Equal("Monitor 'func1-test-test' voted 'ScaleIn'", logs[2].FormattedMessage); - Assert.Equal("Monitor 'testscalemonitor1' voted 'ScaleOut'", logs[3].FormattedMessage); - Assert.Equal("Monitor 'testscalemonitor1' voted 'ScaleIn'", logs[4].FormattedMessage); - Assert.Equal(ScaleVote.ScaleOut, status.Vote); - Assert.Equal(null, status.TargetWorkerCount); - } - else - { - Assert.Equal("1 target scalers to sample", logs[0].FormattedMessage); - Assert.Equal("Snapshot dynamic concurrency for target scaler 'func1' is '1'", logs[1].FormattedMessage); - Assert.Equal("Target worker count for 'func1' is '2'", logs[2].FormattedMessage); - Assert.Equal(ScaleVote.ScaleIn, status.Vote); - Assert.Equal(2, status.TargetWorkerCount); - } - } - - [Fact] - public async Task GetScaleStatus_MonitorFails_ReturnsExpectedResult() - { - var context = new ScaleStatusContext - { - WorkerCount = 3 - }; - - var mockMonitor1 = new Mock(MockBehavior.Strict); - mockMonitor1.Setup(p => p.GetScaleStatus(It.Is(q => q.WorkerCount == context.WorkerCount))).Returns(new ScaleStatus { Vote = ScaleVote.ScaleIn }); - mockMonitor1.SetupGet(p => p.Descriptor).Returns(new ScaleMonitorDescriptor("testscalemonitor1")); - var mockMonitor2 = new Mock(MockBehavior.Strict); - mockMonitor2.SetupGet(p => p.Descriptor).Returns(new ScaleMonitorDescriptor("testscalemonitor2")); - var exception = new Exception("Kaboom!"); - mockMonitor2.Setup(p => p.GetScaleStatus(It.Is(q => q.WorkerCount == context.WorkerCount))).Throws(exception); - var mockMonitor3 = new Mock(MockBehavior.Strict); - mockMonitor3.Setup(p => p.GetScaleStatus(It.Is(q => q.WorkerCount == context.WorkerCount))).Returns(new ScaleStatus { Vote = ScaleVote.ScaleIn }); - mockMonitor3.SetupGet(p => p.Descriptor).Returns(new ScaleMonitorDescriptor("testscalemonitor3")); - List monitors = new List - { - mockMonitor1.Object, - mockMonitor2.Object, - mockMonitor3.Object - }; - - _monitorManagerMock.Setup(p => p.GetMonitors()).Returns(monitors); - - var monitorMetrics = new Dictionary> - { - { mockMonitor1.Object, new List() }, - { mockMonitor2.Object, new List() }, - { mockMonitor3.Object, new List() } - }; - _metricsRepositoryMock.Setup(p => p.ReadMetricsAsync(It.IsAny>())).ReturnsAsync(monitorMetrics); - - var status = await _scaleManager.GetScaleStatusAsync(context); - - var logs = _loggerProvider.GetAllLogMessages(); - Assert.Equal("Computing scale status (WorkerCount=3)", logs[0].FormattedMessage); - Assert.Equal("3 scale monitors to sample", logs[1].FormattedMessage); - Assert.Equal("Monitor 'testscalemonitor1' voted 'ScaleIn'", logs[2].FormattedMessage); - Assert.Equal("Failed to query scale status for monitor 'testscalemonitor2'.", logs[3].FormattedMessage); - Assert.Same(exception, logs[3].Exception); - Assert.Equal("Monitor 'testscalemonitor3' voted 'ScaleIn'", logs[4].FormattedMessage); - - Assert.Equal(null, status.TargetWorkerCount); - Assert.Equal(ScaleVote.ScaleIn, status.Vote); - } - - [Fact] - public async Task GetScaleStatus_TargetScalerFails_ReturnsExpectedResult() - { - var context = new ScaleStatusContext - { - WorkerCount = 3 - }; - - var targetScaler1 = new TestTargetScaler { Result = new TargetScalerResult { TargetWorkerCount = 3 }, TargetScalerDescriptor = new TargetScalerDescriptor("func1") }; - var targetScaler2 = new TestTargetScaler2 { Result = new TargetScalerResult { TargetWorkerCount = 1 }, TargetScalerDescriptor = new TargetScalerDescriptor("func2") }; - var targetScaler3 = new TestTargetScaler { Result = new TargetScalerResult { TargetWorkerCount = -3 }, TargetScalerDescriptor = new TargetScalerDescriptor("func3") }; - List targetScalers = new List - { - targetScaler1, - targetScaler2, - targetScaler3 - }; - _targetScalerManagerMock.Setup(p => p.GetTargetScalers()).Returns(targetScalers); - - _environment.SetEnvironmentVariable(EnvironmentSettingNames.TargetBaseScalingEnabled, "1"); - - string assemblyName = Assembly.GetExecutingAssembly().GetName().Name; - _functionsHostingConfigOptions.Value.Features[assemblyName] = "1"; - - var status = await _scaleManager.GetScaleStatusAsync(context); - - var logs = _loggerProvider.GetAllLogMessages(); - Assert.Equal("3 target scalers to sample", logs[0].FormattedMessage); - Assert.Equal($"Snapshot dynamic concurrency for target scaler 'func1' is '1'", logs[1].FormattedMessage); - Assert.Equal("Target worker count for 'func1' is '3'", logs[2].FormattedMessage); - Assert.Equal("Failed to query scale result for target scaler 'func2'.", logs[3].FormattedMessage); - Assert.Same("test", logs[3].Exception.Message); - Assert.Equal("Target worker count for 'func3' is '-3'", logs[4].FormattedMessage); - - Assert.Equal(3, status.TargetWorkerCount); - Assert.Equal(ScaleVote.None, status.Vote); - } - - [Theory] - [InlineData(0, 0, 0, ScaleVote.None)] - [InlineData(1, 0, 0, ScaleVote.ScaleIn)] - [InlineData(1, 1, 3, ScaleVote.ScaleOut)] - [InlineData(0, 0, 1, ScaleVote.None)] - [InlineData(1, 0, 1, ScaleVote.ScaleIn)] - [InlineData(5, 0, 3, ScaleVote.ScaleIn)] - public void GetAggregateScaleVote_ReturnsExpectedResult(int workerCount, int numScaleOutVotes, int numScaleInVotes, ScaleVote expected) - { - var context = new ScaleStatusContext - { - WorkerCount = workerCount - }; - List votes = new List(); - for (int i = 0; i < numScaleOutVotes; i++) - { - votes.Add(ScaleVote.ScaleOut); - } - for (int i = 0; i < numScaleInVotes; i++) - { - votes.Add(ScaleVote.ScaleIn); - } - var vote = FunctionsScaleManager.GetAggregateScaleVote(votes, context, _testLogger); - Assert.Equal(expected, vote); - } - - [Theory] - [InlineData(false, true, 1, 0)] - [InlineData(true, false, 1, 0)] - [InlineData(true, true, 0, 1)] - public void GetScalersToSample_Returns_Expected(bool targetBaseScalingEnabled, bool triggerEabled, int expectedScaleMonitorCount, int expectedTargetScalerCount) - { - List scaleMonitors = new List - { - new TestScaleMonitor("func1-test-test", "func1"), - }; - Mock scaleMonitorManagerMock = new Mock(MockBehavior.Strict); - scaleMonitorManagerMock.Setup(x => x.GetMonitors()).Returns(scaleMonitors); - - List targetScalers = new List - { - new TestTargetScaler() - { - TargetScalerDescriptor = new TargetScalerDescriptor("func1") - } - }; - Mock targetScalerManagerMock = new Mock(MockBehavior.Strict); - targetScalerManagerMock.Setup(x => x.GetTargetScalers()).Returns(targetScalers); - - string assemblyName = Assembly.GetExecutingAssembly().GetName().Name; - _functionsHostingConfigOptions.Value.Features[assemblyName] = triggerEabled ? "1" : null; - - TestEnvironment env = new TestEnvironment(); - if (targetBaseScalingEnabled) - { - env.SetEnvironmentVariable(EnvironmentSettingNames.TargetBaseScalingEnabled, "1"); - } - - FunctionsScaleManager manager = new FunctionsScaleManager(scaleMonitorManagerMock.Object, _metricsRepositoryMock.Object, - targetScalerManagerMock.Object, _concurrencyStatusRepositoryMock.Object, _functionsHostingConfigOptions, - env, _loggerFactory); - - manager.GetScalersToSample(out List scaleMonitorsToProcess, out List targetScalesToProcess); - - Assert.Equal(scaleMonitorsToProcess.Count(), expectedScaleMonitorCount); - Assert.Equal(targetScalesToProcess.Count(), expectedTargetScalerCount); - } - - [Fact] - public async Task GetScalersToSample_FallsBackToMonitor_OnTargetScalerError() - { - List scaleMonitors = new List - { - new TestScaleMonitor("function1-test-test", "function1") - }; - Mock scaleMonitorManagerMock = new Mock(MockBehavior.Strict); - scaleMonitorManagerMock.Setup(x => x.GetMonitors()).Returns(scaleMonitors); - - List targetScalers = new List - { - new FaultyTargetScaler() - { - TargetScalerDescriptor = new TargetScalerDescriptor("function1") - }, - new TestTargetScaler2() - { - TargetScalerDescriptor = new TargetScalerDescriptor("function2") - } - }; - Mock targetScalerManagerMock = new Mock(MockBehavior.Strict); - targetScalerManagerMock.Setup(x => x.GetTargetScalers()).Returns(targetScalers); - - string assemblyName = Assembly.GetExecutingAssembly().GetName().Name; - _functionsHostingConfigOptions.Value.Features[assemblyName] = "1"; - - TestEnvironment env = new TestEnvironment(); - env.SetEnvironmentVariable(EnvironmentSettingNames.TargetBaseScalingEnabled, "1"); - - FunctionsScaleManager manager = new FunctionsScaleManager(scaleMonitorManagerMock.Object, _metricsRepositoryMock.Object, - targetScalerManagerMock.Object, _concurrencyStatusRepositoryMock.Object, _functionsHostingConfigOptions, - env, _loggerFactory); - - var context = new ScaleStatusContext() - { - WorkerCount = 1 - }; - - manager.GetScalersToSample(out List monitors1, out List scalers1); - Assert.Equal(monitors1.Count(), 0); - Assert.Equal(scalers1.Count(), 2); - ScaleStatusResult resutl1 = await manager.GetScaleStatusAsync(context); - - Assert.Equal(resutl1.TargetWorkerCount, 1); - Assert.Equal(resutl1.Vote, ScaleVote.None); - var logs = _loggerProvider.GetAllLogMessages().Select(x => x.FormattedMessage).ToArray(); - Assert.Single(logs.Where(x => x == "Unable to use target based scaling for Function 'function1'. Metrics monitoring will be used.")); - _loggerProvider.ClearAllLogMessages(); - - manager.GetScalersToSample(out List monitors2, out List scalers2); - Assert.Equal(monitors2.Count(), 1); - Assert.Equal(scalers2.Count(), 1); - ScaleStatusResult resutl2 = await manager.GetScaleStatusAsync(context); - Assert.Equal(resutl2.TargetWorkerCount, null); - Assert.Equal(resutl2.Vote, ScaleVote.ScaleIn); - logs = _loggerProvider.GetAllLogMessages().Select(x => x.FormattedMessage).ToArray(); - Assert.Empty(logs.Where(x => x == "Unable to use target based scaling for Function 'function1'. Metrics monitoring will be used.")); - } - } -} diff --git a/test/WebJobs.Script.Tests/Scale/FunctionsScaleMonitorServiceTests.cs b/test/WebJobs.Script.Tests/Scale/FunctionsScaleMonitorServiceTests.cs index 1c7c66bc32..babd2383be 100644 --- a/test/WebJobs.Script.Tests/Scale/FunctionsScaleMonitorServiceTests.cs +++ b/test/WebJobs.Script.Tests/Scale/FunctionsScaleMonitorServiceTests.cs @@ -28,7 +28,6 @@ public class FunctionsScaleMonitorServiceTests private List _monitors; private List _scalers; private bool _isPrimaryHost; - private IOptions _functionsHostingConfigOptions; public FunctionsScaleMonitorServiceTests() { @@ -43,17 +42,16 @@ public FunctionsScaleMonitorServiceTests() _loggerProvider = new TestLoggerProvider(); ILoggerFactory loggerFactory = new LoggerFactory(); loggerFactory.AddProvider(_loggerProvider); - var scaleOptions = new ScaleOptions(TimeSpan.FromMilliseconds(50)); + var scaleOptions = new ScaleOptions(); + scaleOptions.ScaleMetricsSampleInterval = TimeSpan.FromSeconds(1); var options = new OptionsWrapper(scaleOptions); _environment.SetEnvironmentVariable(EnvironmentSettingNames.FunctionsRuntimeScaleMonitoringEnabled, "1"); - Mock functionsScaleManagerMock = new Mock(); + Mock functionsScaleManagerMock = new Mock(); functionsScaleManagerMock.Setup(x => x.GetScalersToSample(out _monitors, out _scalers)); - _functionsHostingConfigOptions = Options.Create(new FunctionsHostingConfigOptions()); - - _monitor = new FunctionsScaleMonitorService(functionsScaleManagerMock.Object, _metricsRepository, _primaryHostStateProviderMock.Object, _environment, loggerFactory, options); + _monitor = new FunctionsScaleMonitorService(functionsScaleManagerMock.Object, _metricsRepository, loggerFactory, _primaryHostStateProviderMock.Object, _environment, options); } [Fact] @@ -107,7 +105,7 @@ public async Task OnTimer_DoesNotSample_WhenNotPrimaryHost() var logs = _loggerProvider.GetAllLogMessages().ToArray(); Assert.Equal(1, logs.Length); - Assert.Equal("Runtime scale monitoring is enabled.", logs[0].FormattedMessage); + Assert.Equal("Scale monitor service started is started.", logs[0].FormattedMessage); } [Fact] @@ -137,7 +135,7 @@ await TestHelpers.Await(() => var logs = _loggerProvider.GetAllLogMessages().ToArray(); var infoLogs = logs.Where(p => p.Level == LogLevel.Information); - Assert.Equal("Runtime scale monitoring is enabled.", logs[0].FormattedMessage); + Assert.Equal("Scale monitor service started is started.", logs[0].FormattedMessage); Assert.Equal("Taking metrics samples for 1 monitor(s).", logs[1].FormattedMessage); Assert.True(logs[2].FormattedMessage.StartsWith("Scale metrics sample for monitor 'testscalemonitor1': {\"Count\":10,")); @@ -186,7 +184,7 @@ await TestHelpers.Await(() => var logs = _loggerProvider.GetAllLogMessages().ToArray(); var infoLogs = logs.Where(p => p.Level == LogLevel.Information); - Assert.Equal("Runtime scale monitoring is enabled.", logs[0].FormattedMessage); + Assert.Equal("Scale monitor service started is started.", logs[0].FormattedMessage); Assert.Equal("Taking metrics samples for 2 monitor(s).", logs[1].FormattedMessage); // verify the failure logs for the failing monitor