Scaler apis

build/common.props

@@ -1,8 +1,8 @@
 <Project>
   <PropertyGroup>
     <!-- Packages can have independent versions and only increment when released -->
-    <Version>3.0.36$(VersionSuffix)</Version>
-    <HostStorageVersion>5.0.0-beta.2$(VersionSuffix)</HostStorageVersion>
+    <Version>3.0.37$(VersionSuffix)</Version>
+    <HostStorageVersion>5.0.0$(VersionSuffix)</HostStorageVersion>
     <LoggingVersion>4.0.3$(VersionSuffix)</LoggingVersion>
 
     <TargetFramework>netstandard2.0</TargetFramework>

sample/SampleHost/SampleHost.csproj

@@ -15,7 +15,7 @@
   <ItemGroup>
     <PackageReference Include="Microsoft.Azure.WebJobs.Extensions.EventHubs" Version="3.0.6" />
     <PackageReference Include="Microsoft.Azure.WebJobs.Extensions.ServiceBus" Version="3.1.1" />
-    <PackageReference Include="Microsoft.Azure.WebJobs.Extensions.Storage" Version="5.0.0-beta.5" />
+    <PackageReference Include="Microsoft.Azure.WebJobs.Extensions.Storage" Version="5.0.0" />
     <PackageReference Include="Microsoft.Extensions.Configuration.CommandLine" Version="2.1.1" />
     <PackageReference Include="Microsoft.Extensions.Logging" Version="2.1.1" />
     <PackageReference Include="Microsoft.Extensions.Logging.Console" Version="2.1.1" />

src/Microsoft.Azure.WebJobs.Host.Storage/StorageServiceCollectionExtensions.cs

@@ -36,6 +36,13 @@ public static void AddAzureStorageCoreServices(this IServiceCollection services)
             services.AddSingleton<IConcurrencyStatusRepository, BlobStorageConcurrencyStatusRepository>();
         }
 
+        public static void AddAzureStorageScaleServices(this IServiceCollection services)
+        {
+            services.TryAddEnumerable(ServiceDescriptor.Transient<IConfigureOptions<JobHostInternalStorageOptions>, CoreWebJobsOptionsSetup<JobHostInternalStorageOptions>>());
+            services.TryAddSingleton<IAzureBlobStorageProvider, AzureStorageProvider>();
+            services.AddSingleton<IConcurrencyStatusRepository, BlobStorageConcurrencyStatusRepository>();
+        }
+
         // This is only called if the host didn't already provide an implementation
         private static IDistributedLockManager Create(IServiceProvider provider)
         {

src/Microsoft.Azure.WebJobs.Host/Constants.cs

@@ -15,5 +15,6 @@ internal static class Constants
         public const string AzureWebsiteInstanceId = "WEBSITE_INSTANCE_ID";
         public const string AzureWebsiteContainerName = "CONTAINER_NAME";
         public const string DateTimeFormatString = "yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fffK";
+        public const string HostingConfigSectionName = "HostingConfig";
     }
 }

src/Microsoft.Azure.WebJobs.Host/Hosting/WebJobsHostBuilderExtensions.cs

@@ -4,6 +4,7 @@
 using System;
 using System.Linq;
 using Microsoft.Azure.WebJobs;
+using Microsoft.Azure.WebJobs.Host.Scale;
 using Microsoft.Azure.WebJobs.Hosting;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.Configuration.EnvironmentVariables;
@@ -135,6 +136,26 @@ public static IHostBuilder ConfigureWebJobs(this IHostBuilder builder, Action<Ho
             return builder;
         }
 
+        /// <summary>
+        /// Configures the specified <see cref="IHostBuilder"/> as a scale manager host. This method is used for internal infrastructure only.
+        /// </summary>
+        /// <param name="builder">The <see cref="IHostBuilder"/> to configure.</param>
+        /// <param name="configure">Configuration action to perform as part of service configuration.</param>
+        /// <param name="configureScaleOptions">Configuration action for <see cref="ScaleOptions"/>.</param>
+        /// <returns>The <see cref="IHostBuilder"/>.</returns>
+        public static IHostBuilder ConfigureWebJobsScale(this IHostBuilder builder,
+            Action<HostBuilderContext, IWebJobsBuilder> configure,
+            Action<ScaleOptions> configureScaleOptions)
+        {
+            builder.ConfigureServices((context, services) =>
+            {
+                IWebJobsBuilder webJobsBuilder = services.AddWebJobsScale(configureScaleOptions);
+                configure?.Invoke(context, webJobsBuilder);
+            });
+
+            return builder;
+        }
+
         private static IConfigurationBuilder TryAddDefaultConfigurationSources(this IConfigurationBuilder config)
         {
             if (!config.Sources.OfType<JsonConfigurationSource>().Any(p => string.Equals(p.Path, "appsettings.json", StringComparison.OrdinalIgnoreCase)))

src/Microsoft.Azure.WebJobs.Host/Hosting/WebJobsServiceCollectionExtensions.cs

@@ -32,6 +32,7 @@ public static class WebJobsServiceCollectionExtensions
     {
         private const string SingletonConfigSectionName = "Singleton";
         private const string ConcurrencyConfigSectionName = "Concurrency";
+        private const string ScaleConfigSectionName = "Scale";
 
         /// <summary>
         /// Adds the WebJobs services to the provided <see cref="IServiceCollection"/>.
@@ -87,8 +88,9 @@ public static IWebJobsBuilder AddWebJobs(this IServiceCollection services, Actio
             services.TryAddSingleton<IFunctionInstanceLogger, FunctionInstanceLogger>();
             services.TryAddSingleton<IHostInstanceLogger, NullHostInstanceLogger>();
             services.TryAddSingleton<IDistributedLockManager, InMemoryDistributedLockManager>();
-            services.TryAddSingleton<IScaleMonitorManager, ScaleMonitorManager>();
-            services.TryAddSingleton<ITargetScalerManager, TargetScalerManager>();
+
+            services.AddCommonScaleServices();
+            services.TryAddSingleton<IScaleMetricsRepository, NullScaleMetricsRepository>();
 
             services.AddSingleton<IPrimaryHostStateProvider, PrimaryHostStateProvider>();
             services.AddSingleton<IHostedService, PrimaryHostCoordinator>();
@@ -116,12 +118,7 @@ public static IWebJobsBuilder AddWebJobs(this IServiceCollection services, Actio
             services.AddSingleton(typeof(IWebJobsExtensionConfiguration<>), typeof(WebJobsExtensionConfiguration<>));
 
             // Options logging
-            services.AddTransient(typeof(OptionsFactory<>));
-            services.AddTransient(typeof(IOptionsFactory<>), typeof(WebJobsOptionsFactory<>));
-            services.AddSingleton<IOptionsLoggingSource, OptionsLoggingSource>();
-            services.AddSingleton<IHostedService, OptionsLoggingService>();
-            services.AddSingleton<IOptionsFormatter<LoggerFilterOptions>, LoggerFilterOptionsFormatter>();
-
+            services.AddOptionsLogging();
             // Concurrency management
             services.TryAddSingleton<IConcurrencyStatusRepository, NullConcurrencyStatusRepository>();
             services.TryAddSingleton<IHostProcessMonitor, DefaultHostProcessMonitor>();
@@ -159,5 +156,58 @@ public static IWebJobsBuilder AddWebJobs(this IServiceCollection services, Actio
 
             return builder;
         }
+
+        /// <summary>
+        /// Adds the WebJobs scale services to the provided <see cref="IServiceCollection"/>.
+        /// </summary>
+        /// <param name="services"></param>
+        /// <returns></returns>
+        public static IWebJobsBuilder AddWebJobsScale(this IServiceCollection services, Action<ScaleOptions> configure)
+        {
+            if (services == null)
+            {
+                throw new ArgumentNullException(nameof(services));
+            }
+
+            services.AddCommonScaleServices();
+            services.TryAddSingleton<IPrimaryHostStateProvider>(new PrimaryHostStateProvider() { IsPrimary = true });
+
+            if (configure != null)
+            {
+                services.Configure(configure);
+            }
+
+            services.AddOptionsLogging();
+
+            var builder = new WebJobsBuilder(services);
+            return builder;
+        }
+
+        /// <summary>
+        /// Adds scale services common to both normal WebJobs and ScaleHost scenarios.
+        /// </summary>
+        /// <param name="services"></param>
+        private static void AddCommonScaleServices(this IServiceCollection services)
+        {
+            services.AddOptions<ScaleOptions>()
+                .Configure<IConfiguration>((options, config) =>
+                {
+                    var section = config.GetWebJobsRootConfiguration().GetSection(ScaleConfigSectionName);
+                    section.Bind(options);
+                });
+            services.TryAddSingleton<IScaleMonitorManager, ScaleMonitorManager>();
+            services.TryAddSingleton<ITargetScalerManager, TargetScalerManager>();
+            services.TryAddSingleton<IScaleStatusProvider, ScaleManager>();
+            services.AddHostedService<ScaleMonitorService>();
+        }
+
+        private static void AddOptionsLogging(this IServiceCollection services)
+        {
+            services.AddTransient(typeof(OptionsFactory<>));
+            services.AddTransient(typeof(IOptionsFactory<>), typeof(WebJobsOptionsFactory<>));
+            services.AddSingleton<IOptionsLoggingSource, OptionsLoggingSource>();
+            services.AddSingleton<IHostedService, OptionsLoggingService>();
+            services.AddSingleton<IOptionsFormatter<LoggerFilterOptions>, LoggerFilterOptionsFormatter>();
+        }
     }
 }

src/Microsoft.Azure.WebJobs.Host/Scale/AggregateScaleStatus.cs

@@ -0,0 +1,33 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+using System.Collections.Generic;
+
+namespace Microsoft.Azure.WebJobs.Host.Scale
+{
+    /// <summary>
+    /// Represents the aggregate scale status across all functions being monitored by the host.
+    /// </summary>
+    public class AggregateScaleStatus 
+    {
+        /// <summary>
+        /// Gets or sets the aggregate scale vote.
+        /// </summary>
+        public ScaleVote Vote { get; set; }
+
+        /// <summary>
+        /// Gets or sets the aggregate target worker count.
+        /// </summary>
+        public int? TargetWorkerCount { get; set; }
+
+        /// <summary>
+        /// Gets or sets the individual <see cref="ScaleStatus"/>s for all functions being monitored.
+        /// </summary>
+        public IDictionary<string, ScaleStatus> FunctionScaleStatuses { get; set; }
+
+        /// <summary>
+        /// Gets or sets the individual <see cref="TargetScalerResult"/>s for all functions being monitored.
+        /// </summary>
+        public IDictionary<string, TargetScalerResult> FunctionTargetScalerResults { get; set; }
+    }
+}

src/Microsoft.Azure.WebJobs.Host/Scale/IScaleMetricsRepository.cs

@@ -0,0 +1,28 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+using System.Collections.Generic;
+using System.Threading.Tasks;
+
+namespace Microsoft.Azure.WebJobs.Host.Scale
+{
+    /// <summary>
+    /// Interface defining methods for reading/writing scale metrics to a persistent store.
+    /// </summary>
+    public interface IScaleMetricsRepository
+    {
+        /// <summary>
+        /// Persist the metrics for each monitor.
+        /// </summary>
+        /// <param name="monitorMetrics">The collection of metrics for each monitor.</param>
+        /// <returns>A task.</returns>
+        Task WriteMetricsAsync(IDictionary<IScaleMonitor, ScaleMetrics> monitorMetrics);
+
+        /// <summary>
+        /// Read the metrics.
+        /// </summary>
+        /// <param name="monitors">The current collection of monitors.</param>
+        /// <returns>Map of metrics per monitor.</returns>
+        Task<IDictionary<IScaleMonitor, IList<ScaleMetrics>>> ReadMetricsAsync(IEnumerable<IScaleMonitor> monitors);
+    }
+}

src/Microsoft.Azure.WebJobs.Host/Scale/IScaleStatusProvider.cs

@@ -0,0 +1,21 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Microsoft.Azure.WebJobs.Host.Scale
+{
+    /// <summary>
+    /// Interface for providing aggregate scale status across all functions being monitored by the host.
+    /// </summary>
+    public interface IScaleStatusProvider
+    {
+        /// <summary>
+        /// Gets the scale status for all functions being monitored by the host.
+        /// </summary>
+        /// <param name="context">The <see cref="ScaleStatusContext"/>.</param>
+        /// <returns>A task that returns the <see cref="AggregateScaleStatus"/>.</returns>
+        Task<AggregateScaleStatus> GetScaleStatusAsync(ScaleStatusContext context);
+    }
+}

src/Microsoft.Azure.WebJobs.Host/Scale/NullScaleMetricsRepository.cs

@@ -0,0 +1,23 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+using System.Collections.Generic;
+using System.Threading.Tasks;
+
+namespace Microsoft.Azure.WebJobs.Host.Scale
+{
+    internal class NullScaleMetricsRepository : IScaleMetricsRepository
+    {
+        private IDictionary<IScaleMonitor, IList<ScaleMetrics>> _emptyMetrics = new Dictionary<IScaleMonitor, IList<ScaleMetrics>>();
+
+        public Task WriteMetricsAsync(IDictionary<IScaleMonitor, ScaleMetrics> monitorMetrics)
+        {
+            return Task.CompletedTask;
+        }
+
+        public Task<IDictionary<IScaleMonitor, IList<ScaleMetrics>>> ReadMetricsAsync(IEnumerable<IScaleMonitor> monitors)
+        {
+            return Task.FromResult((IDictionary<IScaleMonitor, IList<ScaleMetrics>>)_emptyMetrics);
+        }
+    }
+}