Basic load balancing (#78)

samples/ReverseProxy.Sample/appsettings.json

@@ -17,9 +17,15 @@
   "ReverseProxy": {
     "Backends": {
       "backend1": {
+        "LoadBalancing": {
+          "Mode": "Random"
+        },
         "Endpoints": {
           "backend1/endpoint1": {
             "Address": "https://localhost:10000/"
+          },
+          "backend1/endpoint2": {
+            "Address": "http://localhost:10010/"
           }
         }
       },

samples/SampleServer/Properties/launchSettings.json

@@ -6,7 +6,7 @@
       "environmentVariables": {
         "ASPNETCORE_ENVIRONMENT": "Development"
       },
-      "applicationUrl": "https://localhost:10000;http://localhost:10001"
+      "applicationUrl": "https://localhost:10000;https://localhost:10001;http://localhost:10010;http://localhost:10011"
     }
   }
 }

src/ReverseProxy.Core/Abstractions/BackendDiscovery/Contract/Backend.cs

@@ -32,7 +32,7 @@ public sealed class Backend : IDeepCloneable<Backend>
         /// <summary>
         /// Load balancing options.
         /// </summary>
-        public LoadBalancingOptions LoadBalancingOptions { get; set; }
+        public LoadBalancingOptions LoadBalancing { get; set; }
 
         /// <summary>
         /// Active health checking options.
@@ -57,7 +57,7 @@ Backend IDeepCloneable<Backend>.DeepClone()
                 CircuitBreakerOptions = CircuitBreakerOptions?.DeepClone(),
                 QuotaOptions = QuotaOptions?.DeepClone(),
                 PartitioningOptions = PartitioningOptions?.DeepClone(),
-                LoadBalancingOptions = LoadBalancingOptions?.DeepClone(),
+                LoadBalancing = LoadBalancing?.DeepClone(),
                 HealthCheckOptions = HealthCheckOptions?.DeepClone(),
                 Endpoints = Endpoints.DeepClone(StringComparer.Ordinal),
                 Metadata = Metadata?.DeepClone(StringComparer.Ordinal),

src/ReverseProxy.Core/Abstractions/BackendDiscovery/Contract/LoadBalancingMode.cs

@@ -0,0 +1,29 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+namespace Microsoft.ReverseProxy.Core.Abstractions
+{
+    /// <summary>
+    /// Load balancing strategies for endpoint selection.
+    /// </summary>
+    public enum LoadBalancingMode
+    {
+        /// <summary>
+        /// Select two random endpoints and then select the one with the least assigned requests.
+        /// This avoids the overhead of LeastRequests and the worst case for Random where it selects a busy endpoint.
+        /// </summary>
+        PowerOfTwoChoices,
+        /// <summary>
+        /// Select the endpoint with the least assigned requests. This requires examining all nodes.
+        /// </summary>
+        LeastRequests,
+        /// <summary>
+        /// Select an endpoint randomly.
+        /// </summary>
+        Random,
+        /// <summary>
+        /// Select the first endpoint without considering load. This is useful for dual endpoint fail-over systems.
+        /// </summary>
+        First,
+    }
+}

src/ReverseProxy.Core/Abstractions/BackendDiscovery/Contract/LoadBalancingOptions.cs

@@ -1,4 +1,4 @@
-// Copyright (c) Microsoft Corporation.
+// Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
 namespace Microsoft.ReverseProxy.Core.Abstractions
@@ -8,10 +8,13 @@ namespace Microsoft.ReverseProxy.Core.Abstractions
     /// </summary>
     public sealed class LoadBalancingOptions
     {
+        public LoadBalancingMode Mode { get; set; }
+
         internal LoadBalancingOptions DeepClone()
         {
             return new LoadBalancingOptions
             {
+                Mode = Mode,
             };
         }
     }

src/ReverseProxy.Core/Configuration/DependencyInjection/BuilderExtensions/Core.cs

@@ -1,6 +1,7 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
+using System;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.ReverseProxy.Common.Abstractions.Telemetry;
 using Microsoft.ReverseProxy.Common.Abstractions.Time;
@@ -12,6 +13,7 @@
 using Microsoft.ReverseProxy.Core.Service.Metrics;
 using Microsoft.ReverseProxy.Core.Service.Proxy;
 using Microsoft.ReverseProxy.Core.Service.Proxy.Infra;
+using Microsoft.ReverseProxy.Utilities;
 
 namespace Microsoft.ReverseProxy.Core.Configuration.DependencyInjection
 {
@@ -71,6 +73,7 @@ public static IReverseProxyBuilder AddProxy(this IReverseProxyBuilder builder)
         {
             builder.Services.AddSingleton<IProxyHttpClientFactoryFactory, ProxyHttpClientFactoryFactory>();
             builder.Services.AddSingleton<ILoadBalancer, LoadBalancer>();
+            builder.Services.AddSingleton<IRandomFactory, RandomFactory>();
             builder.Services.AddSingleton<IHttpProxy, HttpProxy>();
             return builder;
         }

src/ReverseProxy.Core/Middleware/LoadBalancingMiddleware.cs

@@ -41,7 +41,6 @@ public Task Invoke(HttpContext context)
             var endpoints = endpointsFeature?.Endpoints
                 ?? throw new InvalidOperationException("The AvailableBackendEndpoints collection was not set.");
 
-            // TODO: Set defaults properly
             var loadBalancingOptions = backend.Config.Value?.LoadBalancingOptions ?? default;
 
             var endpoint = _operationLogger.Execute(

src/ReverseProxy.Core/Service/Management/ReverseProxyConfigManager.cs

@@ -93,7 +93,7 @@ private void UpdateRuntimeBackends(DynamicConfigRoot config)
                                     port: configBackend.HealthCheckOptions?.Port ?? 0,
                                     path: configBackend.HealthCheckOptions?.Path ?? string.Empty),
                                 new BackendConfig.BackendLoadBalancingOptions(
-                                    mode: BackendConfig.BackendLoadBalancingOptions.LoadBalancingMode.First));
+                                    mode: configBackend.LoadBalancing?.Mode ?? default));
 
                         var currentBackendConfig = backend.Config.Value;
                         if (currentBackendConfig == null ||

src/ReverseProxy.Core/Service/Proxy/LoadBalancer.cs

@@ -5,6 +5,7 @@
 using System.Collections.Generic;
 using Microsoft.ReverseProxy.Core.Abstractions;
 using Microsoft.ReverseProxy.Core.RuntimeModel;
+using Microsoft.ReverseProxy.Utilities;
 
 namespace Microsoft.ReverseProxy.Core.Service.Proxy
 {
@@ -13,24 +14,56 @@ namespace Microsoft.ReverseProxy.Core.Service.Proxy
     /// </summary>
     internal class LoadBalancer : ILoadBalancer
     {
+        private readonly IRandomFactory _randomFactory;
+
+        public LoadBalancer(IRandomFactory randomFactory)
+        {
+            _randomFactory = randomFactory;
+        }
+
         public EndpointInfo PickEndpoint(
             IReadOnlyList<EndpointInfo> endpoints,
             in BackendConfig.BackendLoadBalancingOptions loadBalancingOptions)
         {
+            var endpointCount = endpoints.Count;
+            if (endpointCount == 0)
+            {
+                return null;
+            }
+
+            if (endpointCount == 1)
+            {
+                return endpoints[0];
+            }
+
             switch (loadBalancingOptions.Mode)
             {
-                case BackendConfig.BackendLoadBalancingOptions.LoadBalancingMode.First:
-                    // TODO: Remove, this is a silly load balancing mode
-                    if (endpoints.Count == 0)
+                case LoadBalancingMode.First:
+                    return endpoints[0];
+                case LoadBalancingMode.Random:
+                    var random = _randomFactory.CreateRandomInstance();
+                    return endpoints[random.Next(endpointCount)];
+                case LoadBalancingMode.PowerOfTwoChoices:
+                    // Pick two, and then return the least busy. This avoids the effort of searching the whole list, but
+                    // still avoids overloading a single endpoint.
+                    var random1 = _randomFactory.CreateRandomInstance();
+                    var firstEndpoint = endpoints[random1.Next(endpointCount)];
+                    var secondEndpoint = endpoints[random1.Next(endpointCount)];
+                    return (firstEndpoint.ConcurrencyCounter.Value <= secondEndpoint.ConcurrencyCounter.Value) ? firstEndpoint : secondEndpoint;
+                case LoadBalancingMode.LeastRequests:
+                    var leastRequestsEndpoint = endpoints[0];
+                    var leastRequestsCount = leastRequestsEndpoint.ConcurrencyCounter.Value;
+                    for (var i = 1; i < endpointCount; i++)
                     {
-                        return null;
+                        var endpoint = endpoints[i];
+                        var endpointRequestCount = endpoint.ConcurrencyCounter.Value;
+                        if (endpointRequestCount < leastRequestsCount)
+                        {
+                            leastRequestsEndpoint = endpoint;
+                            leastRequestsCount = endpointRequestCount;
+                        }
                     }
-
-                    return endpoints[0];
-                case BackendConfig.BackendLoadBalancingOptions.LoadBalancingMode.Random:
-                    throw new NotImplementedException();
-                case BackendConfig.BackendLoadBalancingOptions.LoadBalancingMode.PowerOfTwoChoices:
-                    throw new NotImplementedException();
+                    return leastRequestsEndpoint;
                 default:
                     throw new NotSupportedException($"Load balancing mode '{loadBalancingOptions.Mode}' is not supported.");
             }

src/ReverseProxy.Core/Service/RuntimeModel/BackendConfig.cs

@@ -2,6 +2,7 @@
 // Licensed under the MIT License.
 
 using System;
+using Microsoft.ReverseProxy.Core.Abstractions;
 
 namespace Microsoft.ReverseProxy.Core.RuntimeModel
 {
@@ -80,13 +81,6 @@ public BackendLoadBalancingOptions(LoadBalancingMode mode)
                 Mode = mode;
             }
 
-            public enum LoadBalancingMode
-            {
-                First,
-                Random,
-                PowerOfTwoChoices,
-            }
-
             public LoadBalancingMode Mode { get; }
         }
     }

src/ReverseProxy.Utilities/Utils/IRandomFactory.cs

@@ -1,6 +1,8 @@
-// Copyright (c) Microsoft Corporation.
+// Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
+using System;
+
 namespace Microsoft.ReverseProxy.Utilities
 {
     /// <summary>
@@ -12,6 +14,6 @@ public interface IRandomFactory
         /// <summary>
         /// Create a instance of random class.
         /// </summary>
-        IRandom CreateRandomInstance();
+        Random CreateRandomInstance();
     }
 }

src/ReverseProxy.Utilities/Utils/RandomFactory.cs

@@ -1,13 +1,15 @@
-// Copyright (c) Microsoft Corporation.
+// Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
+using System;
+
 namespace Microsoft.ReverseProxy.Utilities
 {
     /// <inheritdoc/>
     public class RandomFactory : IRandomFactory
     {
         /// <inheritdoc/>
-        public IRandom CreateRandomInstance()
+        public Random CreateRandomInstance()
         {
             return ThreadStaticRandom.Instance;
         }

src/ReverseProxy.Utilities/Utils/ThreadStaticRandom.cs

@@ -1,4 +1,4 @@
-// Copyright (c) Microsoft Corporation.
+// Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
 using System;
@@ -10,44 +10,8 @@ namespace Microsoft.ReverseProxy.Utilities
     /// </summary>
     public class ThreadStaticRandom
     {
-        /// <summary>
-        /// This is the shared instance of <see cref="RandomWrapper"/> that would be used to generate a seed value for the Thread static instance.
-        /// </summary>
-        private static readonly Lazy<RandomWrapper> _globalRandom = new Lazy<RandomWrapper>(() => new RandomWrapper(new Random()));
-
-        /// <summary>
-        /// This instance of <see cref="RandomWrapper"/> is unique to each thread.
-        /// </summary>
         [ThreadStatic]
-        private static RandomWrapper _threadLocalRandom = null;
-
-        /// <summary>
-        /// Gets the a thread safe instance of <see cref="RandomWrapper"/>.
-        /// </summary>
-        public static IRandom Instance
-        {
-            get
-            {
-                var currentInstance = _threadLocalRandom;
-
-                // Check if for the current thread the seed has already been established. If not then lock on the global random instance to generate a seed value
-                if (currentInstance == null)
-                {
-                    int seedForThreadLocalInstance;
-
-                    lock (_globalRandom.Value)
-                    {
-                        seedForThreadLocalInstance = _globalRandom.Value.Next();
-                    }
-
-                    // Initialize the current instance with the seed
-                    var random = new Random(seedForThreadLocalInstance);
-                    currentInstance = new RandomWrapper(random);
-                    _threadLocalRandom = currentInstance;
-                }
-
-                return currentInstance;
-            }
-        }
+        private static Random t_inst;
+        public static Random Instance => t_inst ??= new Random();
     }
 }