diff --git a/src/WebJobs.Script.WebHost/Controllers/HostController.cs b/src/WebJobs.Script.WebHost/Controllers/HostController.cs index 8fb3f5137f..2360900c19 100644 --- a/src/WebJobs.Script.WebHost/Controllers/HostController.cs +++ b/src/WebJobs.Script.WebHost/Controllers/HostController.cs @@ -317,17 +317,17 @@ public async Task GetScaleStatus([FromBody] ScaleStatusContext co } // TEMP: Once https://github.com/Azure/azure-functions-host/issues/5161 is fixed, we should take - // FunctionsScaleManager as a parameter. - if (Utility.TryGetHostService(scriptHostManager, out FunctionsScaleManager scaleManager)) + // IScaleStatusProvider as a parameter. + if (Utility.TryGetHostService(scriptHostManager, out IScaleStatusProvider scaleStatusProvider)) { - var scaleStatus = await scaleManager.GetScaleStatusAsync(context); + var scaleStatus = await scaleStatusProvider.GetScaleStatusAsync(context); return new ObjectResult(scaleStatus); } else { // This case should never happen. Because this action is marked RequiresRunningHost, // it's only invoked when the host is running, and if it's running, we'll have access - // to the FunctionsScaleManager. + // to the IScaleStatusProvider. return StatusCode(StatusCodes.Status503ServiceUnavailable); } } diff --git a/src/WebJobs.Script.WebHost/DependencyInjection/DependencyValidator/DependencyValidator.cs b/src/WebJobs.Script.WebHost/DependencyInjection/DependencyValidator/DependencyValidator.cs index f9a9251942..4fe404433e 100644 --- a/src/WebJobs.Script.WebHost/DependencyInjection/DependencyValidator/DependencyValidator.cs +++ b/src/WebJobs.Script.WebHost/DependencyInjection/DependencyValidator/DependencyValidator.cs @@ -48,7 +48,6 @@ private static ExpectedDependencyBuilder CreateExpectedDependencies() .Expect() .Expect() .Expect() - .Optional() .Optional() // Used by powershell. .Optional() // Missing when host is offline. .Optional() // Conditionally registered. @@ -56,7 +55,8 @@ private static ExpectedDependencyBuilder CreateExpectedDependencies() .OptionalExternal("Microsoft.Extensions.Diagnostics.HealthChecks.HealthCheckPublisherHostedService", "Microsoft.Extensions.Diagnostics.HealthChecks", "adb9793829ddae60") // Popularly-registered by Health Check Monitor. .OptionalExternal("OpenTelemetry.Extensions.Hosting.Implementation.TelemetryHostedService", "OpenTelemetry.Extensions.Hosting", "7bd6737fe5b67e3c") // Enable OpenTelemetry.Net instrumentation library .OptionalExternal("Microsoft.Azure.WebJobs.Hosting.PrimaryHostCoordinator", "Microsoft.Azure.WebJobs.Host", "31bf3856ad364e35") - .OptionalExternal("Microsoft.Azure.WebJobs.Host.Scale.ConcurrencyManagerService", "Microsoft.Azure.WebJobs.Host", "31bf3856ad364e35"); + .OptionalExternal("Microsoft.Azure.WebJobs.Host.Scale.ConcurrencyManagerService", "Microsoft.Azure.WebJobs.Host", "31bf3856ad364e35") + .OptionalExternal("Microsoft.Azure.WebJobs.Host.Scale.ScaleMonitorService", "Microsoft.Azure.WebJobs.Host", "31bf3856ad364e35"); expected.ExpectSubcollection() .Expect() diff --git a/src/WebJobs.Script.WebHost/WebJobs.Script.WebHost.csproj b/src/WebJobs.Script.WebHost/WebJobs.Script.WebHost.csproj index 68d14a629b..7ee8742ebb 100644 --- a/src/WebJobs.Script.WebHost/WebJobs.Script.WebHost.csproj +++ b/src/WebJobs.Script.WebHost/WebJobs.Script.WebHost.csproj @@ -77,7 +77,7 @@ - + diff --git a/src/WebJobs.Script.WebHost/WebScriptHostBuilderExtension.cs b/src/WebJobs.Script.WebHost/WebScriptHostBuilderExtension.cs index 0af93a489e..5d69006697 100644 --- a/src/WebJobs.Script.WebHost/WebScriptHostBuilderExtension.cs +++ b/src/WebJobs.Script.WebHost/WebScriptHostBuilderExtension.cs @@ -131,7 +131,7 @@ public static IHostBuilder AddWebScriptHost(this IHostBuilder builder, IServiceP // EasyAuth must go after CORS, as CORS preflight requests can happen before authentication services.TryAddEnumerable(ServiceDescriptor.Singleton()); } - services.TryAddSingleton(); + services.AddSingleton(); services.TryAddEnumerable(ServiceDescriptor.Singleton()); diff --git a/src/WebJobs.Script/Config/ScaleOptions.cs b/src/WebJobs.Script/Config/ScaleOptions.cs deleted file mode 100644 index c57b49880a..0000000000 --- a/src/WebJobs.Script/Config/ScaleOptions.cs +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the MIT License. See License.txt in the project root for license information. - -using System; -using Microsoft.Azure.WebJobs.Hosting; -using Newtonsoft.Json; -using Newtonsoft.Json.Linq; - -namespace Microsoft.Azure.WebJobs.Script -{ - /// - /// Defines configuration options for runtime scale monitoring. - /// - public class ScaleOptions : IOptionsFormatter - { - private TimeSpan _scaleMetricsMaxAge; - private TimeSpan _scaleMetricsSampleInterval; - - public ScaleOptions() - { - // At the default values, a single monitor will be generating 6 samples per minute - // so at 2 minutes that's 12 samples - // Assume a case of 100 functions in an app, each mapping to a monitor. Thats - // 1200 samples to read from storage on each scale status request. - ScaleMetricsMaxAge = TimeSpan.FromMinutes(2); - ScaleMetricsSampleInterval = TimeSpan.FromSeconds(10); - MetricsPurgeEnabled = true; - } - - // for testing, to allow us to bypass validations - internal ScaleOptions(TimeSpan metricsSampleInterval) : this() - { - _scaleMetricsSampleInterval = metricsSampleInterval; - } - - /// - /// Gets or sets a value indicating the maximum age for metrics. - /// Metrics that exceed this age will not be returned to monitors. - /// - public TimeSpan ScaleMetricsMaxAge - { - get - { - return _scaleMetricsMaxAge; - } - - set - { - if (value < TimeSpan.FromMinutes(1) || value > TimeSpan.FromMinutes(5)) - { - throw new ArgumentOutOfRangeException(nameof(ScaleMetricsMaxAge)); - } - _scaleMetricsMaxAge = value; - } - } - - /// - /// Gets or sets the sampling interval for metrics. - /// - public TimeSpan ScaleMetricsSampleInterval - { - get - { - return _scaleMetricsSampleInterval; - } - - set - { - if (value < TimeSpan.FromSeconds(1) || value > TimeSpan.FromSeconds(30)) - { - throw new ArgumentOutOfRangeException(nameof(ScaleMetricsSampleInterval)); - } - _scaleMetricsSampleInterval = value; - } - } - - /// - /// Gets or sets a value indicating whether old metrics data - /// will be auto purged. - /// - public bool MetricsPurgeEnabled { get; set; } - - public string Format() - { - var options = new JObject - { - { nameof(ScaleMetricsMaxAge), ScaleMetricsMaxAge }, - { nameof(ScaleMetricsSampleInterval), ScaleMetricsSampleInterval } - }; - - return options.ToString(Formatting.Indented); - } - } -} diff --git a/src/WebJobs.Script/Config/ScaleOptionsSetup.cs b/src/WebJobs.Script/Config/ScaleOptionsSetup.cs new file mode 100644 index 0000000000..48cf81f280 --- /dev/null +++ b/src/WebJobs.Script/Config/ScaleOptionsSetup.cs @@ -0,0 +1,25 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using Microsoft.Azure.WebJobs.Host.Scale; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Options; + +namespace Microsoft.Azure.WebJobs.Script.Configuration +{ + internal class ScaleOptionsSetup : IConfigureOptions + { + private readonly IEnvironment _environment; + + public ScaleOptionsSetup(IEnvironment environment) + { + _environment = environment; + } + + public void Configure(ScaleOptions options) + { + options.IsRuntimeScalingEnabled = _environment.IsRuntimeScaleMonitoringEnabled(); + options.IsTargetScalingEnabled = _environment.IsTargetBasedScalingEnabled(); + } + } +} diff --git a/src/WebJobs.Script/Scale/FunctionsScaleManager.cs b/src/WebJobs.Script/Scale/FunctionsScaleManager.cs deleted file mode 100644 index 6cb3f081cb..0000000000 --- a/src/WebJobs.Script/Scale/FunctionsScaleManager.cs +++ /dev/null @@ -1,294 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the MIT License. See License.txt in the project root for license information. - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Reflection; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Azure.AppService.Proxy.Client; -using Microsoft.Azure.WebJobs.Host.Scale; -using Microsoft.Azure.WebJobs.Script.Config; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; - -namespace Microsoft.Azure.WebJobs.Script.Scale -{ - /// - /// Manages scale monitoring operations. - /// - public class FunctionsScaleManager - { - private readonly IScaleMonitorManager _monitorManager; - private readonly IScaleMetricsRepository _metricsRepository; - private readonly ITargetScalerManager _targetScalerManager; - private readonly IConcurrencyStatusRepository _concurrencyStatusRepository; - private readonly IOptions _functionsHostingConfigOptions; - private readonly IEnvironment _environment; - private readonly ILogger _logger; - private readonly HashSet _targetScalersInError; - - // for mock testing only - public FunctionsScaleManager() - { - } - - public FunctionsScaleManager( - IScaleMonitorManager monitorManager, - IScaleMetricsRepository metricsRepository, - ITargetScalerManager targetScalerManager, - IConcurrencyStatusRepository concurrencyStatusRepository, - IOptions functionsHostingConfigOptions, - IEnvironment environment, - ILoggerFactory loggerFactory) - { - _monitorManager = monitorManager; - _metricsRepository = metricsRepository; - _targetScalerManager = targetScalerManager; - _concurrencyStatusRepository = concurrencyStatusRepository; - _functionsHostingConfigOptions = functionsHostingConfigOptions; - _environment = environment; - _logger = loggerFactory.CreateLogger(); - _targetScalersInError = new HashSet(); - } - - /// - /// Get the current scale status (vote) by querying all active monitors for their - /// scale status. - /// - /// The context to use for the scale decision. - /// The scale vote. - public virtual async Task GetScaleStatusAsync(ScaleStatusContext context) - { - GetScalersToSample(out List scaleMonitorsToProcess, out List targetScalersToProcess); - - var scaleMonitorVotes = await GetScaleMonitorsResultAsync(context, scaleMonitorsToProcess); - var targetScalerVotes = await GetTargetScalersResultAsync(context, targetScalersToProcess); - - return new ScaleStatusResult - { - Vote = GetAggregateScaleVote(scaleMonitorVotes.Union(targetScalerVotes.Select(x => x.Vote)), context, _logger), - TargetWorkerCount = targetScalerVotes.Any() ? targetScalerVotes.Max(x => x.TargetWorkerCount) : null - }; - } - - private async Task> GetScaleMonitorsResultAsync(ScaleStatusContext context, IEnumerable scaleMonitorsToProcess) - { - List votes = new List(); - if (scaleMonitorsToProcess.Any()) - { - // get the collection of current metrics for each monitor - var monitorMetrics = await _metricsRepository.ReadMetricsAsync(scaleMonitorsToProcess); - - _logger.LogDebug($"Computing scale status (WorkerCount={context.WorkerCount})"); - _logger.LogDebug($"{monitorMetrics.Count} scale monitors to sample"); - - // for each monitor, ask it to return its scale status (vote) based on - // the metrics and context info (e.g. worker count) - foreach (var pair in monitorMetrics) - { - var monitor = pair.Key; - var metrics = pair.Value; - - try - { - // create a new context instance to avoid modifying the - // incoming context - var scaleStatusContext = new ScaleStatusContext - { - WorkerCount = context.WorkerCount, - Metrics = metrics - }; - var result = monitor.GetScaleStatus(scaleStatusContext); - - _logger.LogDebug($"Monitor '{monitor.Descriptor.Id}' voted '{result.Vote.ToString()}'"); - votes.Add(result.Vote); - } - catch (Exception exc) when (!exc.IsFatal()) - { - // if a particular monitor fails, log and continue - _logger.LogError(exc, $"Failed to query scale status for monitor '{monitor.Descriptor.Id}'."); - } - } - } - else - { - // no monitors registered - // this can happen if the host is offline - } - - return votes; - } - - private async Task> GetTargetScalersResultAsync(ScaleStatusContext context, IEnumerable targetScalersToProcess) - { - List targetScaleVotes = new List(); - - if (targetScalersToProcess.Any()) - { - _logger.LogDebug($"{targetScalersToProcess.Count()} target scalers to sample"); - HostConcurrencySnapshot snapshot = null; - try - { - snapshot = await _concurrencyStatusRepository.ReadAsync(CancellationToken.None); - } - catch (Exception exc) when (!exc.IsFatal()) - { - _logger.LogError(exc, $"Failed to read concurrency status repository"); - } - - foreach (var targetScaler in targetScalersToProcess) - { - try - { - TargetScalerContext targetScaleStatusContext = new TargetScalerContext(); - if (snapshot != null) - { - if (snapshot.FunctionSnapshots.TryGetValue(targetScaler.TargetScalerDescriptor.FunctionId, out var functionSnapshot)) - { - targetScaleStatusContext.InstanceConcurrency = functionSnapshot.Concurrency; - _logger.LogDebug($"Snapshot dynamic concurrency for target scaler '{targetScaler.TargetScalerDescriptor.FunctionId}' is '{functionSnapshot.Concurrency}'"); - } - } - TargetScalerResult result = null; - try - { - result = await targetScaler.GetScaleResultAsync(targetScaleStatusContext); - } - catch (NotSupportedException ex) - { - string targetScalerUniqueId = GetTargetScalerFunctionUniqueId(targetScaler); - _logger.LogWarning($"Unable to use target based scaling for Function '{targetScaler.TargetScalerDescriptor.FunctionId}'. Metrics monitoring will be used.", ex); - _targetScalersInError.Add(targetScalerUniqueId); - - // Adding ScaleVote.None vote - result = new TargetScalerResult - { - TargetWorkerCount = context.WorkerCount - }; - } - _logger.LogDebug($"Target worker count for '{targetScaler.TargetScalerDescriptor.FunctionId}' is '{result.TargetWorkerCount}'"); - - ScaleVote vote = ScaleVote.None; - if (context.WorkerCount > result.TargetWorkerCount) - { - vote = ScaleVote.ScaleIn; - } - else if (context.WorkerCount < result.TargetWorkerCount) - { - vote = ScaleVote.ScaleOut; - } - - targetScaleVotes.Add(new TargetScalerVote - { - TargetWorkerCount = result.TargetWorkerCount, - Vote = vote - }); - } - catch (Exception exc) when (!exc.IsFatal()) - { - // if a particular target scaler fails, log and continue - _logger.LogError(exc, $"Failed to query scale result for target scaler '{targetScaler.TargetScalerDescriptor.FunctionId}'."); - } - } - } - return targetScaleVotes; - } - - internal static ScaleVote GetAggregateScaleVote(IEnumerable votes, ScaleStatusContext context, ILogger logger) - { - ScaleVote vote = ScaleVote.None; - if (votes.Any()) - { - // aggregate all the votes into a single vote - if (votes.Any(p => p == ScaleVote.ScaleOut)) - { - // scale out if at least 1 monitor requires it - logger.LogDebug("Scaling out based on votes"); - vote = ScaleVote.ScaleOut; - } - else if (context.WorkerCount > 0 && votes.All(p => p == ScaleVote.ScaleIn)) - { - // scale in only if all monitors vote scale in - logger.LogDebug("Scaling in based on votes"); - vote = ScaleVote.ScaleIn; - } - } - else if (context.WorkerCount > 0) - { - // if no functions exist or are enabled we'll scale in - logger.LogDebug("No enabled functions or scale votes so scaling in"); - vote = ScaleVote.ScaleIn; - } - - return vote; - } - - /// - /// Returns scale monitors and target scalers we want to use based on the configuration. - /// Scaler monitor will be ignored if a target scaler is defined in the same extensions assembly and TBS is enabled. - /// - /// Scale monitor to process. - /// Target scaler to process. - public virtual void GetScalersToSample( - out List scaleMonitorsToSample, - out List targetScalersToSample) - { - var scaleMonitors = _monitorManager.GetMonitors(); - var targetScalers = _targetScalerManager.GetTargetScalers(); - - scaleMonitorsToSample = new List(); - targetScalersToSample = new List(); - - // Check if TBS enabled on app level - if (_environment.IsTargetBasedScalingEnabled()) - { - HashSet targetScalerFunctions = new HashSet(); - foreach (var scaler in targetScalers) - { - string scalerUniqueId = GetTargetScalerFunctionUniqueId(scaler); - if (!_targetScalersInError.Contains(scalerUniqueId)) - { - string assemblyName = GetAssemblyName(scaler.GetType()); - string flag = _functionsHostingConfigOptions.Value.GetFeature(assemblyName); - if (flag == "1") - { - targetScalersToSample.Add(scaler); - targetScalerFunctions.Add(scalerUniqueId); - } - } - } - - foreach (var monitor in scaleMonitors) - { - string monitorUniqueId = GetScaleMonitorFunctionUniqueId(monitor); - // Check if target based scaler exists for the function - if (!targetScalerFunctions.Contains(monitorUniqueId)) - { - scaleMonitorsToSample.Add(monitor); - } - } - } - else - { - scaleMonitorsToSample.AddRange(scaleMonitors); - } - } - - private string GetTargetScalerFunctionUniqueId(ITargetScaler scaler) - { - return $"{GetAssemblyName(scaler.GetType())}-{scaler.TargetScalerDescriptor.FunctionId}"; - } - - private string GetScaleMonitorFunctionUniqueId(IScaleMonitor monitor) - { - return $"{GetAssemblyName(monitor.GetType())}-{monitor.Descriptor.FunctionId}"; - } - - private string GetAssemblyName(Type type) - { - return type.Assembly.GetName().Name; - } - } -} diff --git a/src/WebJobs.Script/Scale/FunctionsScaleMonitorService.cs b/src/WebJobs.Script/Scale/FunctionsScaleMonitorService.cs deleted file mode 100644 index 12ba8c48c2..0000000000 --- a/src/WebJobs.Script/Scale/FunctionsScaleMonitorService.cs +++ /dev/null @@ -1,159 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the MIT License. See License.txt in the project root for license information. - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Azure.WebJobs.Host.Scale; -using Microsoft.Azure.WebJobs.Hosting; -using Microsoft.Azure.WebJobs.Script.Config; -using Microsoft.Extensions.Hosting; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; -using Newtonsoft.Json; - -namespace Microsoft.Azure.WebJobs.Script.Scale -{ - /// - /// Service responsible for taking periodic scale metrics samples and persisting them. - /// - public class FunctionsScaleMonitorService : IHostedService, IDisposable - { - private readonly IPrimaryHostStateProvider _primaryHostStateProvider; - private readonly IScaleMetricsRepository _metricsRepository; - private readonly IEnvironment _environment; - private readonly ILogger _logger; - private readonly Timer _timer; - private readonly TimeSpan _interval; - private readonly ScaleOptions _scaleOptions; - private readonly FunctionsScaleManager _functionsScaleManager; - private bool _disposed; - - public FunctionsScaleMonitorService(FunctionsScaleManager functionsScaleManager, IScaleMetricsRepository metricsRepository, IPrimaryHostStateProvider primaryHostStateProvider, IEnvironment environment, ILoggerFactory loggerFactory, IOptions scaleOptions) - { - _metricsRepository = metricsRepository; - _primaryHostStateProvider = primaryHostStateProvider; - _environment = environment; - _logger = loggerFactory.CreateLogger(); - _scaleOptions = scaleOptions.Value; - - _interval = _scaleOptions.ScaleMetricsSampleInterval; - _timer = new Timer(OnTimer, null, Timeout.Infinite, Timeout.Infinite); - _functionsScaleManager = functionsScaleManager; - } - - public Task StartAsync(CancellationToken cancellationToken) - { - if (_environment.IsRuntimeScaleMonitoringEnabled()) - { - _logger.LogInformation("Runtime scale monitoring is enabled."); - - // start the timer by setting the due time - SetTimerInterval((int)_interval.TotalMilliseconds); - } - - return Task.CompletedTask; - } - - public Task StopAsync(CancellationToken cancellationToken) - { - // stop the timer if it has been started - _timer.Change(Timeout.Infinite, Timeout.Infinite); - - return Task.CompletedTask; - } - - private async void OnTimer(object state) - { - if (_primaryHostStateProvider.IsPrimary) - { - await TakeMetricsSamplesAsync(); - } - - SetTimerInterval((int)_interval.TotalMilliseconds); - } - - internal async Task TakeMetricsSamplesAsync() - { - try - { - _functionsScaleManager.GetScalersToSample(out List scaleMonitorsToProcess, out List targetScalersToProcess); - - if (scaleMonitorsToProcess.Any()) - { - _logger.LogDebug($"Taking metrics samples for {scaleMonitorsToProcess.Count()} monitor(s)."); - - var metricsMap = new Dictionary(); - foreach (var monitor in scaleMonitorsToProcess) - { - ScaleMetrics metrics = null; - try - { - // take a metrics sample for each monitor - metrics = await monitor.GetMetricsAsync(); - metricsMap[monitor] = metrics; - - // log the metrics json to provide visibility into monitor activity - var json = JsonConvert.SerializeObject(metrics); - _logger.LogDebug($"Scale metrics sample for monitor '{monitor.Descriptor.Id}': {json}"); - } - catch (Exception exc) when (!exc.IsFatal()) - { - // if a particular monitor fails, log and continue - _logger.LogError(exc, $"Failed to collect scale metrics sample for monitor '{monitor.Descriptor.Id}'."); - } - } - - if (metricsMap.Count > 0) - { - // persist the metrics samples - await _metricsRepository.WriteMetricsAsync(metricsMap); - } - } - } - catch (Exception exc) when (!exc.IsFatal()) - { - _logger.LogError(exc, "Failed to collect/persist scale metrics."); - } - } - - private void SetTimerInterval(int dueTime) - { - if (!_disposed) - { - var timer = _timer; - if (timer != null) - { - try - { - _timer.Change(dueTime, Timeout.Infinite); - } - catch (ObjectDisposedException) - { - // might race with dispose - } - } - } - } - - private void Dispose(bool disposing) - { - if (!_disposed) - { - if (disposing) - { - _timer.Dispose(); - } - - _disposed = true; - } - } - - public void Dispose() - { - Dispose(true); - } - } -} diff --git a/src/WebJobs.Script/Scale/IScaleMetricsRepository.cs b/src/WebJobs.Script/Scale/IScaleMetricsRepository.cs deleted file mode 100644 index 3aa6c6e692..0000000000 --- a/src/WebJobs.Script/Scale/IScaleMetricsRepository.cs +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the MIT License. See License.txt in the project root for license information. - -using System.Collections.Generic; -using System.Threading.Tasks; -using Microsoft.Azure.WebJobs.Host.Scale; - -namespace Microsoft.Azure.WebJobs.Script.Scale -{ - /// - /// Interface defining methods for reading/writing scale metrics to a persistent store. - /// - public interface IScaleMetricsRepository - { - /// - /// Persist the metrics for each monitor. - /// - /// The collection of metrics for each monitor. - /// A task. - Task WriteMetricsAsync(IDictionary monitorMetrics); - - /// - /// Read the metrics. - /// - /// The current collection of monitors. - /// Map of metrics per monitor. - Task>> ReadMetricsAsync(IEnumerable monitors); - } -} diff --git a/src/WebJobs.Script/Scale/ScaleStatusResult.cs b/src/WebJobs.Script/Scale/ScaleStatusResult.cs deleted file mode 100644 index 1ccd2b8dd5..0000000000 --- a/src/WebJobs.Script/Scale/ScaleStatusResult.cs +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the MIT License. See License.txt in the project root for license information. - -using Microsoft.Azure.WebJobs.Host.Scale; - -namespace Microsoft.Azure.WebJobs.Script.Scale -{ - /// - /// Represents an aggregate scale status result returned from - /// - /// - /// We use a separate model type for this rather than reusing to decouple these contracts. This type - /// is used as the contract with ScaleController and is serialized externally. is the contract between - /// the host and binding extensions. - /// - public class ScaleStatusResult - { - public ScaleVote Vote { get; set; } - - public int? TargetWorkerCount { get; set; } - } -} diff --git a/src/WebJobs.Script/ScriptHostBuilderExtensions.cs b/src/WebJobs.Script/ScriptHostBuilderExtensions.cs index 63f7b045d6..e7131b8723 100644 --- a/src/WebJobs.Script/ScriptHostBuilderExtensions.cs +++ b/src/WebJobs.Script/ScriptHostBuilderExtensions.cs @@ -12,6 +12,7 @@ using Microsoft.ApplicationInsights.WindowsServer.TelemetryChannel; using Microsoft.Azure.WebJobs.Host; using Microsoft.Azure.WebJobs.Host.Executors; +using Microsoft.Azure.WebJobs.Host.Scale; using Microsoft.Azure.WebJobs.Hosting; using Microsoft.Azure.WebJobs.Logging; using Microsoft.Azure.WebJobs.Logging.ApplicationInsights; @@ -102,6 +103,12 @@ public static IHostBuilder AddScriptHost(this IHostBuilder builder, { configBuilder.Add(new HostJsonFileConfigurationSource(applicationOptions, SystemEnvironment.Instance, loggerFactory, metricsLogger)); } + // Adding hosting config into job host configuration + IConfiguration scriptHostConfiguration = applicationOptions.RootServiceProvider.GetService(); + if (scriptHostConfiguration != null) + { + configBuilder.AddConfiguration(scriptHostConfiguration.GetSection(ScriptConstants.FunctionsHostingConfigSectionName)); + } }); // WebJobs configuration @@ -306,13 +313,7 @@ public static IHostBuilder AddScriptHostCore(this IHostBuilder builder, ScriptAp .GetSection(ConfigurationSectionNames.Aggregator) .Bind(o); }); - services.AddOptions() - .Configure((o, c) => - { - c.GetSection(ConfigurationSectionNames.JobHost) - .GetSection(ConfigurationSectionNames.Scale) - .Bind(o); - }); + services.ConfigureOptions(); services.AddSingleton(); @@ -328,12 +329,6 @@ public static IHostBuilder AddScriptHostCore(this IHostBuilder builder, ScriptAp services.TryAddEnumerable(ServiceDescriptor.Singleton()); - if (SystemEnvironment.Instance.IsRuntimeScaleMonitoringEnabled()) - { - services.TryAddEnumerable(ServiceDescriptor.Singleton()); - } - services.TryAddSingleton(); - services.AddSingleton(); }); diff --git a/src/WebJobs.Script/WebJobs.Script.csproj b/src/WebJobs.Script/WebJobs.Script.csproj index bd6e38d970..1580705022 100644 --- a/src/WebJobs.Script/WebJobs.Script.csproj +++ b/src/WebJobs.Script/WebJobs.Script.csproj @@ -48,7 +48,7 @@ - + diff --git a/test/CSharpPrecompiledTestProjects/WebJobsStartupTests/Function1.cs b/test/CSharpPrecompiledTestProjects/WebJobsStartupTests/Function1.cs index 72bed6c78b..59e65434f4 100644 --- a/test/CSharpPrecompiledTestProjects/WebJobsStartupTests/Function1.cs +++ b/test/CSharpPrecompiledTestProjects/WebJobsStartupTests/Function1.cs @@ -93,7 +93,7 @@ private static bool ValidateConfig(IConfiguration _config) { if (_config is ConfigurationRoot root) { - if (root.Providers.Count() != 7) + if (root.Providers.Count() != 8) { return false; } @@ -104,6 +104,7 @@ private static bool ValidateConfig(IConfiguration _config) root.Providers.ElementAt(i++) is ChainedConfigurationProvider && root.Providers.ElementAt(i++) is MemoryConfigurationProvider && root.Providers.ElementAt(i++).GetType().Name.StartsWith("HostJsonFile") && + root.Providers.ElementAt(i++) is ChainedConfigurationProvider && root.Providers.ElementAt(i++) is JsonConfigurationProvider && root.Providers.ElementAt(i++) is EnvironmentVariablesConfigurationProvider && root.Providers.ElementAt(i++) is MemoryConfigurationProvider && // From Startup.cs diff --git a/test/WebJobs.Script.Tests/Configuration/ScaleMonitorOptionsSetupTests.cs b/test/WebJobs.Script.Tests/Configuration/ScaleMonitorOptionsSetupTests.cs new file mode 100644 index 0000000000..c9ef19df11 --- /dev/null +++ b/test/WebJobs.Script.Tests/Configuration/ScaleMonitorOptionsSetupTests.cs @@ -0,0 +1,45 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using Microsoft.Azure.WebJobs.Host.Scale; +using Microsoft.Azure.WebJobs.Script.Configuration; +using Xunit; + +namespace Microsoft.Azure.WebJobs.Script.Tests.Configuration +{ + public class ScaleMonitorOptionsSetupTests + { + [Theory] + [InlineData(true, true)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(false, false)] + public void ScaleOptionsSetup_ConfiguresExpectedDefaults(bool runtimeScaleMonitoringEnabled, bool targetBaseScalingEnabled) + { + var testEnvironment = new TestEnvironment(); + + if (runtimeScaleMonitoringEnabled) + { + testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.FunctionsRuntimeScaleMonitoringEnabled, "1"); + } + if (!targetBaseScalingEnabled) + { + testEnvironment.SetEnvironmentVariable(EnvironmentSettingNames.TargetBaseScalingEnabled, "0"); + } + + ScaleOptionsSetup setup = new ScaleOptionsSetup(testEnvironment); + ScaleOptions options = new ScaleOptions(); + + setup.Configure(options); + + if (runtimeScaleMonitoringEnabled) + { + Assert.Equal(runtimeScaleMonitoringEnabled, options.IsRuntimeScalingEnabled); + } + if (!targetBaseScalingEnabled) + { + Assert.Equal(targetBaseScalingEnabled, options.IsTargetScalingEnabled); + } + } + } +} diff --git a/test/WebJobs.Script.Tests/Controllers/Admin/HostControllerTests.cs b/test/WebJobs.Script.Tests/Controllers/Admin/HostControllerTests.cs index d112dc8a6e..16795d8dc4 100644 --- a/test/WebJobs.Script.Tests/Controllers/Admin/HostControllerTests.cs +++ b/test/WebJobs.Script.Tests/Controllers/Admin/HostControllerTests.cs @@ -37,6 +37,8 @@ public class HostControllerTests private readonly Mock _mockHostPerformanceManager; private readonly HostHealthMonitorOptions _hostHealthMonitorOptions; private readonly ScriptApplicationHostOptions _applicationHostOptions; + private readonly Mock _scaleStatusProvider; + private readonly LoggerFactory _loggerFactory; public HostControllerTests() { @@ -46,8 +48,8 @@ public HostControllerTests() var optionsWrapper = new OptionsWrapper(_applicationHostOptions); var loggerProvider = new TestLoggerProvider(); - var loggerFactory = new LoggerFactory(); - loggerFactory.AddProvider(loggerProvider); + _loggerFactory = new LoggerFactory(); + _loggerFactory.AddProvider(loggerProvider); _mockEnvironment = new Mock(MockBehavior.Strict); _mockEnvironment.Setup(p => p.GetEnvironmentVariable(It.IsAny())).Returns(null); _mockScriptHostManager = new Mock(MockBehavior.Strict); @@ -59,13 +61,22 @@ public HostControllerTests() _hostHealthMonitorOptions = new HostHealthMonitorOptions(); var wrappedHealthMonitorOptions = new OptionsWrapper(_hostHealthMonitorOptions); _mockHostPerformanceManager = new Mock(_mockEnvironment.Object, wrappedHealthMonitorOptions, mockServiceProvider.Object); - _hostController = new HostController(optionsWrapper, loggerFactory, _mockEnvironment.Object, _mockScriptHostManager.Object, _functionsSyncManager.Object, _mockHostPerformanceManager.Object); + _hostController = new HostController(optionsWrapper, _loggerFactory, _mockEnvironment.Object, _mockScriptHostManager.Object, _functionsSyncManager.Object, _mockHostPerformanceManager.Object); _appOfflineFilePath = Path.Combine(_scriptPath, ScriptConstants.AppOfflineFileName); if (File.Exists(_appOfflineFilePath)) { File.Delete(_appOfflineFilePath); } + + _scaleStatusProvider = new Mock(MockBehavior.Strict); + _scaleStatusProvider.Setup(p => p.GetScaleStatusAsync(It.IsAny())).Returns(() => + { + return Task.FromResult(new AggregateScaleStatus() + { + Vote = ScaleVote.ScaleIn + }); + }); } [Theory] @@ -148,18 +159,15 @@ public async Task GetScaleStatus_RuntimeScaleModeEnabled_Succeeds() { WorkerCount = 5 }; - var scaleManagerMock = new Mock(MockBehavior.Strict); - var scaleStatusResult = new ScaleStatusResult { Vote = ScaleVote.ScaleOut, TargetWorkerCount = 2 }; - scaleManagerMock.Setup(p => p.GetScaleStatusAsync(context)).ReturnsAsync(scaleStatusResult); var scriptHostManagerMock = new Mock(MockBehavior.Strict); var serviceProviderMock = scriptHostManagerMock.As(); - serviceProviderMock.Setup(p => p.GetService(typeof(FunctionsScaleManager))).Returns(scaleManagerMock.Object); + serviceProviderMock.Setup(p => p.GetService(typeof(IScaleStatusProvider))).Returns(_scaleStatusProvider.Object); var result = (ObjectResult)(await _hostController.GetScaleStatus(context, scriptHostManagerMock.Object)); - Assert.Same(result.Value, scaleStatusResult); + Assert.Equal(((AggregateScaleStatus)result.Value).Vote, ScaleVote.ScaleIn); } [Fact] - public async Task GetScaleStatus_FunctionsScaleManager_Null_ReturnsServiceUnavailable() + public async Task GetScaleStatus_IScaleStatusProvider_Null_ReturnsServiceUnavailable() { _mockEnvironment.Setup(p => p.GetEnvironmentVariable(EnvironmentSettingNames.FunctionsRuntimeScaleMonitoringEnabled)).Returns("1"); @@ -169,7 +177,7 @@ public async Task GetScaleStatus_FunctionsScaleManager_Null_ReturnsServiceUnavai }; var scriptHostManagerMock = new Mock(MockBehavior.Strict); var serviceProviderMock = scriptHostManagerMock.As(); - serviceProviderMock.Setup(p => p.GetService(typeof(FunctionsScaleManager))).Returns(null); + serviceProviderMock.Setup(p => p.GetService(typeof(IScaleStatusProvider))).Returns(null); var result = (StatusCodeResult)(await _hostController.GetScaleStatus(context, scriptHostManagerMock.Object)); Assert.Equal(StatusCodes.Status503ServiceUnavailable, result.StatusCode); @@ -182,10 +190,9 @@ public async Task GetScaleStatus_RuntimeScaleModeNotEnabled_ReturnsBadRequest() { WorkerCount = 5 }; - var scaleManagerMock = new Mock(MockBehavior.Strict); + Mock serviceProviderMock = new Mock(); + serviceProviderMock.Setup(p => p.GetService(typeof(IScaleStatusProvider))).Returns(_scaleStatusProvider.Object); var scriptHostManagerMock = new Mock(MockBehavior.Strict); - var serviceProviderMock = scriptHostManagerMock.As(); - serviceProviderMock.Setup(p => p.GetService(typeof(FunctionsScaleManager))).Returns(scaleManagerMock.Object); var result = (BadRequestObjectResult)(await _hostController.GetScaleStatus(context, scriptHostManagerMock.Object)); Assert.Equal(HttpStatusCode.BadRequest, (HttpStatusCode)result.StatusCode); Assert.Equal("Runtime scale monitoring is not enabled.", result.Value); diff --git a/test/WebJobs.Script.Tests/Scale/FunctionsScaleManagerTests.cs b/test/WebJobs.Script.Tests/Scale/FunctionsScaleManagerTests.cs deleted file mode 100644 index 6f56d232bf..0000000000 --- a/test/WebJobs.Script.Tests/Scale/FunctionsScaleManagerTests.cs +++ /dev/null @@ -1,387 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the MIT License. See License.txt in the project root for license information. - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Reflection; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Azure.WebJobs.Host.Scale; -using Microsoft.Azure.WebJobs.Script.Config; -using Microsoft.Azure.WebJobs.Script.Scale; -using Microsoft.Azure.WebJobs.Script.Tests.Description.DotNet; -using Microsoft.Azure.WebJobs.Script.WebHost.Filters; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; -using Microsoft.WebJobs.Script.Tests; -using Moq; -using Xunit; - -namespace Microsoft.Azure.WebJobs.Script.Tests.Scale -{ - public class FunctionsScaleManagerTests - { - private readonly FunctionsScaleManager _scaleManager; - private readonly Mock _monitorManagerMock; - private readonly Mock _metricsRepositoryMock; - private readonly Mock _targetScalerManagerMock; - private readonly Mock _concurrencyStatusRepositoryMock; - private readonly ILoggerFactory _loggerFactory; - private readonly TestLoggerProvider _loggerProvider; - private readonly List _monitors; - private readonly List _targetScalers; - private readonly TestEnvironment _environment; - private readonly ILogger _testLogger; - private IOptions _functionsHostingConfigOptions; - - public FunctionsScaleManagerTests() - { - _monitors = new List(); - _targetScalers = new List(); - _loggerProvider = new TestLoggerProvider(); - _loggerFactory = new LoggerFactory(); - _loggerFactory.AddProvider(_loggerProvider); - _testLogger = _loggerFactory.CreateLogger("Test"); - - _monitorManagerMock = new Mock(MockBehavior.Strict); - _monitorManagerMock.Setup(p => p.GetMonitors()).Returns(() => _monitors); - _metricsRepositoryMock = new Mock(MockBehavior.Strict); - _metricsRepositoryMock.Setup(x => x.ReadMetricsAsync(It.IsAny>())).ReturnsAsync(new Dictionary>()); - _targetScalerManagerMock = new Mock(MockBehavior.Strict); - _targetScalerManagerMock.Setup(p => p.GetTargetScalers()).Returns(() => _targetScalers); - _concurrencyStatusRepositoryMock = new Mock(MockBehavior.Strict); - _concurrencyStatusRepositoryMock.Setup(p => p.ReadAsync(It.IsAny())).ReturnsAsync( - new HostConcurrencySnapshot() - { - FunctionSnapshots = new Dictionary() - { - { "func1", new FunctionConcurrencySnapshot() { Concurrency = 1 } } - } - }); - - _functionsHostingConfigOptions = Options.Create(new FunctionsHostingConfigOptions()); - - _environment = new TestEnvironment(); - - _scaleManager = new FunctionsScaleManager(_monitorManagerMock.Object, _metricsRepositoryMock.Object, _targetScalerManagerMock.Object, _concurrencyStatusRepositoryMock.Object, - _functionsHostingConfigOptions, _environment, _loggerFactory); - } - - [Theory] - [InlineData(0, ScaleVote.None)] - [InlineData(1, ScaleVote.ScaleIn)] - public async Task GetScaleStatus_NoMonitors_ReturnsExpectedStatus(int workerCount, ScaleVote expected) - { - var context = new ScaleStatusContext - { - WorkerCount = workerCount - }; - var status = await _scaleManager.GetScaleStatusAsync(context); - - Assert.Equal(expected, status.Vote); - } - - [Theory] - [InlineData(true)] - [InlineData(false)] - public async Task GetScaleStatus_ReturnsExpectedResult(bool tbsEnabled) - { - var context = new ScaleStatusContext - { - WorkerCount = 3 - }; - - var monitor1 = new TestScaleMonitor("func1-test-test", "func1"); - monitor1.Status = new ScaleStatus - { - Vote = ScaleVote.ScaleIn - }; - var monitor2 = new TestScaleMonitor1(); - monitor2.Status = new ScaleStatus - { - Vote = ScaleVote.ScaleOut - }; - var monitor3 = new TestScaleMonitor1(); - monitor3.Status = new ScaleStatus - { - Vote = ScaleVote.ScaleIn - }; - - List monitors = new List - { - monitor1 - }; - if (!tbsEnabled) - { - monitors.Add(monitor2); - monitors.Add(monitor3); - } - _monitorManagerMock.Setup(p => p.GetMonitors()).Returns(monitors); - - var monitorMetrics = new Dictionary> - { - { monitor1, new List() }, - { monitor2, new List() }, - { monitor3, new List() } - }; - _metricsRepositoryMock.Setup(p => p.ReadMetricsAsync(It.IsAny>())).ReturnsAsync(monitorMetrics); - - var targetScaler1 = new TestTargetScaler() - { - Result = new TargetScalerResult() - { - TargetWorkerCount = 2 - }, - TargetScalerDescriptor = new TargetScalerDescriptor("func1") - }; - - _targetScalerManagerMock.Setup(p => p.GetTargetScalers()).Returns(new List { targetScaler1 }); - - if (!tbsEnabled) - { - _environment.SetEnvironmentVariable(EnvironmentSettingNames.TargetBaseScalingEnabled, "0"); - } - - string assemblyName = Assembly.GetExecutingAssembly().GetName().Name; - _functionsHostingConfigOptions.Value.Features[assemblyName] = "1"; - - var status = await _scaleManager.GetScaleStatusAsync(context); - - var logs = _loggerProvider.GetAllLogMessages(); - if (!tbsEnabled) - { - Assert.Equal("Computing scale status (WorkerCount=3)", logs[0].FormattedMessage); - Assert.Equal("3 scale monitors to sample", logs[1].FormattedMessage); - Assert.Equal("Monitor 'func1-test-test' voted 'ScaleIn'", logs[2].FormattedMessage); - Assert.Equal("Monitor 'testscalemonitor1' voted 'ScaleOut'", logs[3].FormattedMessage); - Assert.Equal("Monitor 'testscalemonitor1' voted 'ScaleIn'", logs[4].FormattedMessage); - Assert.Equal(ScaleVote.ScaleOut, status.Vote); - Assert.Equal(null, status.TargetWorkerCount); - } - else - { - Assert.Equal("1 target scalers to sample", logs[0].FormattedMessage); - Assert.Equal("Snapshot dynamic concurrency for target scaler 'func1' is '1'", logs[1].FormattedMessage); - Assert.Equal("Target worker count for 'func1' is '2'", logs[2].FormattedMessage); - Assert.Equal(ScaleVote.ScaleIn, status.Vote); - Assert.Equal(2, status.TargetWorkerCount); - } - } - - [Fact] - public async Task GetScaleStatus_MonitorFails_ReturnsExpectedResult() - { - var context = new ScaleStatusContext - { - WorkerCount = 3 - }; - - var mockMonitor1 = new Mock(MockBehavior.Strict); - mockMonitor1.Setup(p => p.GetScaleStatus(It.Is(q => q.WorkerCount == context.WorkerCount))).Returns(new ScaleStatus { Vote = ScaleVote.ScaleIn }); - mockMonitor1.SetupGet(p => p.Descriptor).Returns(new ScaleMonitorDescriptor("testscalemonitor1")); - var mockMonitor2 = new Mock(MockBehavior.Strict); - mockMonitor2.SetupGet(p => p.Descriptor).Returns(new ScaleMonitorDescriptor("testscalemonitor2")); - var exception = new Exception("Kaboom!"); - mockMonitor2.Setup(p => p.GetScaleStatus(It.Is(q => q.WorkerCount == context.WorkerCount))).Throws(exception); - var mockMonitor3 = new Mock(MockBehavior.Strict); - mockMonitor3.Setup(p => p.GetScaleStatus(It.Is(q => q.WorkerCount == context.WorkerCount))).Returns(new ScaleStatus { Vote = ScaleVote.ScaleIn }); - mockMonitor3.SetupGet(p => p.Descriptor).Returns(new ScaleMonitorDescriptor("testscalemonitor3")); - List monitors = new List - { - mockMonitor1.Object, - mockMonitor2.Object, - mockMonitor3.Object - }; - - _monitorManagerMock.Setup(p => p.GetMonitors()).Returns(monitors); - - var monitorMetrics = new Dictionary> - { - { mockMonitor1.Object, new List() }, - { mockMonitor2.Object, new List() }, - { mockMonitor3.Object, new List() } - }; - _metricsRepositoryMock.Setup(p => p.ReadMetricsAsync(It.IsAny>())).ReturnsAsync(monitorMetrics); - - var status = await _scaleManager.GetScaleStatusAsync(context); - - var logs = _loggerProvider.GetAllLogMessages(); - Assert.Equal("Computing scale status (WorkerCount=3)", logs[0].FormattedMessage); - Assert.Equal("3 scale monitors to sample", logs[1].FormattedMessage); - Assert.Equal("Monitor 'testscalemonitor1' voted 'ScaleIn'", logs[2].FormattedMessage); - Assert.Equal("Failed to query scale status for monitor 'testscalemonitor2'.", logs[3].FormattedMessage); - Assert.Same(exception, logs[3].Exception); - Assert.Equal("Monitor 'testscalemonitor3' voted 'ScaleIn'", logs[4].FormattedMessage); - - Assert.Equal(null, status.TargetWorkerCount); - Assert.Equal(ScaleVote.ScaleIn, status.Vote); - } - - [Fact] - public async Task GetScaleStatus_TargetScalerFails_ReturnsExpectedResult() - { - var context = new ScaleStatusContext - { - WorkerCount = 3 - }; - - var targetScaler1 = new TestTargetScaler { Result = new TargetScalerResult { TargetWorkerCount = 3 }, TargetScalerDescriptor = new TargetScalerDescriptor("func1") }; - var targetScaler2 = new TestTargetScaler2 { Result = new TargetScalerResult { TargetWorkerCount = 1 }, TargetScalerDescriptor = new TargetScalerDescriptor("func2") }; - var targetScaler3 = new TestTargetScaler { Result = new TargetScalerResult { TargetWorkerCount = -3 }, TargetScalerDescriptor = new TargetScalerDescriptor("func3") }; - List targetScalers = new List - { - targetScaler1, - targetScaler2, - targetScaler3 - }; - _targetScalerManagerMock.Setup(p => p.GetTargetScalers()).Returns(targetScalers); - - _environment.SetEnvironmentVariable(EnvironmentSettingNames.TargetBaseScalingEnabled, "1"); - - string assemblyName = Assembly.GetExecutingAssembly().GetName().Name; - _functionsHostingConfigOptions.Value.Features[assemblyName] = "1"; - - var status = await _scaleManager.GetScaleStatusAsync(context); - - var logs = _loggerProvider.GetAllLogMessages(); - Assert.Equal("3 target scalers to sample", logs[0].FormattedMessage); - Assert.Equal($"Snapshot dynamic concurrency for target scaler 'func1' is '1'", logs[1].FormattedMessage); - Assert.Equal("Target worker count for 'func1' is '3'", logs[2].FormattedMessage); - Assert.Equal("Failed to query scale result for target scaler 'func2'.", logs[3].FormattedMessage); - Assert.Same("test", logs[3].Exception.Message); - Assert.Equal("Target worker count for 'func3' is '-3'", logs[4].FormattedMessage); - - Assert.Equal(3, status.TargetWorkerCount); - Assert.Equal(ScaleVote.None, status.Vote); - } - - [Theory] - [InlineData(0, 0, 0, ScaleVote.None)] - [InlineData(1, 0, 0, ScaleVote.ScaleIn)] - [InlineData(1, 1, 3, ScaleVote.ScaleOut)] - [InlineData(0, 0, 1, ScaleVote.None)] - [InlineData(1, 0, 1, ScaleVote.ScaleIn)] - [InlineData(5, 0, 3, ScaleVote.ScaleIn)] - public void GetAggregateScaleVote_ReturnsExpectedResult(int workerCount, int numScaleOutVotes, int numScaleInVotes, ScaleVote expected) - { - var context = new ScaleStatusContext - { - WorkerCount = workerCount - }; - List votes = new List(); - for (int i = 0; i < numScaleOutVotes; i++) - { - votes.Add(ScaleVote.ScaleOut); - } - for (int i = 0; i < numScaleInVotes; i++) - { - votes.Add(ScaleVote.ScaleIn); - } - var vote = FunctionsScaleManager.GetAggregateScaleVote(votes, context, _testLogger); - Assert.Equal(expected, vote); - } - - [Theory] - [InlineData(false, true, 1, 0)] - [InlineData(true, false, 1, 0)] - [InlineData(true, true, 0, 1)] - public void GetScalersToSample_Returns_Expected(bool targetBaseScalingEnabled, bool triggerEabled, int expectedScaleMonitorCount, int expectedTargetScalerCount) - { - List scaleMonitors = new List - { - new TestScaleMonitor("func1-test-test", "func1"), - }; - Mock scaleMonitorManagerMock = new Mock(MockBehavior.Strict); - scaleMonitorManagerMock.Setup(x => x.GetMonitors()).Returns(scaleMonitors); - - List targetScalers = new List - { - new TestTargetScaler() - { - TargetScalerDescriptor = new TargetScalerDescriptor("func1") - } - }; - Mock targetScalerManagerMock = new Mock(MockBehavior.Strict); - targetScalerManagerMock.Setup(x => x.GetTargetScalers()).Returns(targetScalers); - - string assemblyName = Assembly.GetExecutingAssembly().GetName().Name; - _functionsHostingConfigOptions.Value.Features[assemblyName] = triggerEabled ? "1" : null; - - TestEnvironment env = new TestEnvironment(); - if (!targetBaseScalingEnabled) - { - env.SetEnvironmentVariable(EnvironmentSettingNames.TargetBaseScalingEnabled, "0"); - } - - FunctionsScaleManager manager = new FunctionsScaleManager(scaleMonitorManagerMock.Object, _metricsRepositoryMock.Object, - targetScalerManagerMock.Object, _concurrencyStatusRepositoryMock.Object, _functionsHostingConfigOptions, - env, _loggerFactory); - - manager.GetScalersToSample(out List scaleMonitorsToProcess, out List targetScalesToProcess); - - Assert.Equal(scaleMonitorsToProcess.Count(), expectedScaleMonitorCount); - Assert.Equal(targetScalesToProcess.Count(), expectedTargetScalerCount); - } - - [Fact] - public async Task GetScalersToSample_FallsBackToMonitor_OnTargetScalerError() - { - List scaleMonitors = new List - { - new TestScaleMonitor("function1-test-test", "function1") - }; - Mock scaleMonitorManagerMock = new Mock(MockBehavior.Strict); - scaleMonitorManagerMock.Setup(x => x.GetMonitors()).Returns(scaleMonitors); - - List targetScalers = new List - { - new FaultyTargetScaler() - { - TargetScalerDescriptor = new TargetScalerDescriptor("function1") - }, - new TestTargetScaler2() - { - TargetScalerDescriptor = new TargetScalerDescriptor("function2") - } - }; - Mock targetScalerManagerMock = new Mock(MockBehavior.Strict); - targetScalerManagerMock.Setup(x => x.GetTargetScalers()).Returns(targetScalers); - - string assemblyName = Assembly.GetExecutingAssembly().GetName().Name; - _functionsHostingConfigOptions.Value.Features[assemblyName] = "1"; - - TestEnvironment env = new TestEnvironment(); - env.SetEnvironmentVariable(EnvironmentSettingNames.TargetBaseScalingEnabled, "1"); - - FunctionsScaleManager manager = new FunctionsScaleManager(scaleMonitorManagerMock.Object, _metricsRepositoryMock.Object, - targetScalerManagerMock.Object, _concurrencyStatusRepositoryMock.Object, _functionsHostingConfigOptions, - env, _loggerFactory); - - var context = new ScaleStatusContext() - { - WorkerCount = 1 - }; - - manager.GetScalersToSample(out List monitors1, out List scalers1); - Assert.Equal(monitors1.Count(), 0); - Assert.Equal(scalers1.Count(), 2); - ScaleStatusResult resutl1 = await manager.GetScaleStatusAsync(context); - - Assert.Equal(resutl1.TargetWorkerCount, 1); - Assert.Equal(resutl1.Vote, ScaleVote.None); - var logs = _loggerProvider.GetAllLogMessages().Select(x => x.FormattedMessage).ToArray(); - Assert.Single(logs.Where(x => x == "Unable to use target based scaling for Function 'function1'. Metrics monitoring will be used.")); - _loggerProvider.ClearAllLogMessages(); - - manager.GetScalersToSample(out List monitors2, out List scalers2); - Assert.Equal(monitors2.Count(), 1); - Assert.Equal(scalers2.Count(), 1); - ScaleStatusResult resutl2 = await manager.GetScaleStatusAsync(context); - Assert.Equal(resutl2.TargetWorkerCount, null); - Assert.Equal(resutl2.Vote, ScaleVote.ScaleIn); - logs = _loggerProvider.GetAllLogMessages().Select(x => x.FormattedMessage).ToArray(); - Assert.Empty(logs.Where(x => x == "Unable to use target based scaling for Function 'function1'. Metrics monitoring will be used.")); - } - } -} diff --git a/test/WebJobs.Script.Tests/Scale/FunctionsScaleMonitorServiceTests.cs b/test/WebJobs.Script.Tests/Scale/FunctionsScaleMonitorServiceTests.cs deleted file mode 100644 index 1c7c66bc32..0000000000 --- a/test/WebJobs.Script.Tests/Scale/FunctionsScaleMonitorServiceTests.cs +++ /dev/null @@ -1,239 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the MIT License. See License.txt in the project root for license information. - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Azure.WebJobs.Host.Scale; -using Microsoft.Azure.WebJobs.Hosting; -using Microsoft.Azure.WebJobs.Script.Config; -using Microsoft.Azure.WebJobs.Script.Scale; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; -using Microsoft.WebJobs.Script.Tests; -using Moq; -using Xunit; - -namespace Microsoft.Azure.WebJobs.Script.Tests.Scale -{ - public class FunctionsScaleMonitorServiceTests - { - private readonly FunctionsScaleMonitorService _monitor; - private readonly TestMetricsRepository _metricsRepository; - private readonly Mock _primaryHostStateProviderMock; - private readonly TestEnvironment _environment; - private readonly TestLoggerProvider _loggerProvider; - private List _monitors; - private List _scalers; - private bool _isPrimaryHost; - private IOptions _functionsHostingConfigOptions; - - public FunctionsScaleMonitorServiceTests() - { - _isPrimaryHost = true; - _monitors = new List(); - _scalers = new List(); - - _metricsRepository = new TestMetricsRepository(); - _primaryHostStateProviderMock = new Mock(MockBehavior.Strict); - _primaryHostStateProviderMock.SetupGet(p => p.IsPrimary).Returns(() => _isPrimaryHost); - _environment = new TestEnvironment(); - _loggerProvider = new TestLoggerProvider(); - ILoggerFactory loggerFactory = new LoggerFactory(); - loggerFactory.AddProvider(_loggerProvider); - var scaleOptions = new ScaleOptions(TimeSpan.FromMilliseconds(50)); - var options = new OptionsWrapper(scaleOptions); - - _environment.SetEnvironmentVariable(EnvironmentSettingNames.FunctionsRuntimeScaleMonitoringEnabled, "1"); - - Mock functionsScaleManagerMock = new Mock(); - functionsScaleManagerMock.Setup(x => x.GetScalersToSample(out _monitors, out _scalers)); - - _functionsHostingConfigOptions = Options.Create(new FunctionsHostingConfigOptions()); - - _monitor = new FunctionsScaleMonitorService(functionsScaleManagerMock.Object, _metricsRepository, _primaryHostStateProviderMock.Object, _environment, loggerFactory, options); - } - - [Fact] - public async Task OnTimer_ExceptionsAreHandled() - { - var monitor = new TestScaleMonitor1 - { - Exception = new Exception("Kaboom!") - }; - _monitors.Add(monitor); - - await _monitor.StartAsync(CancellationToken.None); - - // wait for a few failures to happen - LogMessage[] logs = null; - await TestHelpers.Await(() => - { - logs = _loggerProvider.GetAllLogMessages().Where(p => p.Level == LogLevel.Error).ToArray(); - return logs.Length >= 3; - }); - - Assert.All(logs, - p => - { - Assert.Same(monitor.Exception, p.Exception); - Assert.Equal("Failed to collect scale metrics sample for monitor 'testscalemonitor1'.", p.FormattedMessage); - }); - } - - [Fact] - public async Task StartAsync_RuntimeScaleMonitoringNotEnabled_DoesNotStart() - { - _environment.SetEnvironmentVariable(EnvironmentSettingNames.FunctionsRuntimeScaleMonitoringEnabled, "0"); - - await _monitor.StartAsync(CancellationToken.None); - - Assert.Empty(_loggerProvider.GetAllLogMessages()); - } - - [Fact] - public async Task OnTimer_DoesNotSample_WhenNotPrimaryHost() - { - _isPrimaryHost = false; - - var monitor = new TestScaleMonitor1(); - _monitors.Add(monitor); - - await _monitor.StartAsync(CancellationToken.None); - - await Task.Delay(100); - - var logs = _loggerProvider.GetAllLogMessages().ToArray(); - Assert.Equal(1, logs.Length); - Assert.Equal("Runtime scale monitoring is enabled.", logs[0].FormattedMessage); - } - - [Fact] - public async Task OnTimer_PersistsMetrics() - { - var testMetrics = new List - { - new TestScaleMetrics1 { Count = 10 }, - new TestScaleMetrics1 { Count = 15 }, - new TestScaleMetrics1 { Count = 45 }, - new TestScaleMetrics1 { Count = 50 }, - new TestScaleMetrics1 { Count = 100 } - }; - var monitor1 = new TestScaleMonitor1 - { - Metrics = testMetrics - }; - _monitors.Add(monitor1); - - await _monitor.StartAsync(CancellationToken.None); - - await TestHelpers.Await(() => - { - return _metricsRepository.Count >= 5; - }); - - var logs = _loggerProvider.GetAllLogMessages().ToArray(); - - var infoLogs = logs.Where(p => p.Level == LogLevel.Information); - Assert.Equal("Runtime scale monitoring is enabled.", logs[0].FormattedMessage); - Assert.Equal("Taking metrics samples for 1 monitor(s).", logs[1].FormattedMessage); - Assert.True(logs[2].FormattedMessage.StartsWith("Scale metrics sample for monitor 'testscalemonitor1': {\"Count\":10,")); - - var metricsWritten = _metricsRepository.Metrics[monitor1].Take(5); - Assert.Equal(testMetrics, metricsWritten); - } - - [Fact] - public async Task OnTimer_MonitorFailuresAreHandled() - { - var testMetrics1 = new List - { - new TestScaleMetrics1 { Count = 10 }, - new TestScaleMetrics1 { Count = 15 }, - new TestScaleMetrics1 { Count = 45 }, - new TestScaleMetrics1 { Count = 50 }, - new TestScaleMetrics1 { Count = 100 } - }; - var monitor1 = new TestScaleMonitor1 - { - Exception = new Exception("Kaboom!") - }; - _monitors.Add(monitor1); - - var testMetrics2 = new List - { - new TestScaleMetrics2 { Num = 300 }, - new TestScaleMetrics2 { Num = 350 }, - new TestScaleMetrics2 { Num = 400 }, - new TestScaleMetrics2 { Num = 450 }, - new TestScaleMetrics2 { Num = 500 } - }; - var monitor2 = new TestScaleMonitor2 - { - Metrics = testMetrics2 - }; - _monitors.Add(monitor2); - - await _monitor.StartAsync(CancellationToken.None); - - await TestHelpers.Await(() => - { - return _metricsRepository.Count >= 5; - }); - - var logs = _loggerProvider.GetAllLogMessages().ToArray(); - - var infoLogs = logs.Where(p => p.Level == LogLevel.Information); - Assert.Equal("Runtime scale monitoring is enabled.", logs[0].FormattedMessage); - Assert.Equal("Taking metrics samples for 2 monitor(s).", logs[1].FormattedMessage); - - // verify the failure logs for the failing monitor - Assert.True(logs.Count(p => p.FormattedMessage.Equals($"Failed to collect scale metrics sample for monitor 'testscalemonitor1'.")) >= 5); - - // verify each successful sample is logged - Assert.True(logs.Count(p => p.FormattedMessage.StartsWith($"Scale metrics sample for monitor 'testscalemonitor2'")) >= 5); - - var metricsWritten = _metricsRepository.Metrics[monitor2].Take(5); - Assert.Equal(testMetrics2, metricsWritten); - } - } - - public class TestMetricsRepository : IScaleMetricsRepository - { - private int _count; - - public TestMetricsRepository() - { - _count = 0; - Metrics = new Dictionary>(); - } - - public int Count => _count; - - public IDictionary> Metrics { get; set; } - - public Task>> ReadMetricsAsync(IEnumerable monitors) - { - return Task.FromResult>>(Metrics); - } - - public Task WriteMetricsAsync(IDictionary monitorMetrics) - { - foreach (var pair in monitorMetrics) - { - if (!Metrics.ContainsKey(pair.Key)) - { - Metrics[pair.Key] = new List(); - } - - Metrics[pair.Key].Add(pair.Value); - - Interlocked.Increment(ref _count); - } - - return Task.CompletedTask; - } - } -} diff --git a/test/WebJobs.Script.Tests/WebJobs.Script.Tests.csproj b/test/WebJobs.Script.Tests/WebJobs.Script.Tests.csproj index 0812b1eb11..164427c37e 100644 --- a/test/WebJobs.Script.Tests/WebJobs.Script.Tests.csproj +++ b/test/WebJobs.Script.Tests/WebJobs.Script.Tests.csproj @@ -166,6 +166,7 @@ +