diff --git a/.config/dotnet-tools.json b/.config/dotnet-tools.json index d9c539fb0..646b0d473 100644 --- a/.config/dotnet-tools.json +++ b/.config/dotnet-tools.json @@ -13,6 +13,13 @@ "commands": [ "csmacnz.Coveralls" ] + }, + "dotnet-reportgenerator-globaltool": { + "version": "5.4.7", + "commands": [ + "reportgenerator" + ], + "rollForward": false } } } diff --git a/docs/features/kubernetes.rst b/docs/features/kubernetes.rst index 8f4ee02a1..a4415c66e 100644 --- a/docs/features/kubernetes.rst +++ b/docs/features/kubernetes.rst @@ -179,8 +179,21 @@ The following examples show how to set up a route that will work in Kubernetes. The most important thing is the ``ServiceName`` which is made up of the Kubernetes service name. We also need to set up the ``ServiceDiscoveryProvider`` in ``GlobalConfiguration``. +Regarding global and route configurations, if your downstream service resides in a different namespace, you can override the global setting at the route level by specifying a ``ServiceNamespace``. + +.. code-block:: json + + "Routes": [ + { + "ServiceName": "my-service", + "ServiceNamespace": "my-namespace" + } + ] + +.. _k8s-kube-provider: + ``Kube`` provider -^^^^^^^^^^^^^^^^^ +----------------- The example here shows a typical configuration: @@ -203,7 +216,7 @@ The example here shows a typical configuration: } } -Service deployment in ``Dev`` namespace, and discovery provider type is ``Kube``, you also can set :ref:`k8s-pollkube-provider` type. +Service deployment in ``Dev`` namespace, and discovery provider type is ``Kube``, you also can set :ref:`k8s-pollkube-provider` or :ref:`k8s-watchkube-provider` type. **Note 1**: ``Scheme``, ``Host``, ``Port``, and ``Token`` are not used if ``usePodServiceAccount`` is true when `KubeClient`_ is created from a pod service account. Please refer to the :ref:`k8s-install` section for technical details. @@ -215,7 +228,7 @@ Service deployment in ``Dev`` namespace, and discovery provider type is ``Kube`` .. _k8s-pollkube-provider: ``PollKube`` provider -^^^^^^^^^^^^^^^^^^^^^ +--------------------- You use Ocelot to poll Kubernetes for latest service information rather than per request. If you want to poll Kubernetes for the latest services rather than per request (default behaviour) then you need to set the following configuration: @@ -236,23 +249,81 @@ The polling interval is in milliseconds and tells Ocelot how often to call Kuber We doubt it will matter for most people and polling may give a tiny performance improvement over calling Kubernetes per request. There is no way for Ocelot to work these out for you, except perhaps through a `discussion `_. -Global vs Route levels -^^^^^^^^^^^^^^^^^^^^^^ +.. _k8s-watchkube-provider: -If your downstream service resides in a different namespace, you can override the global setting at the route-level by specifying a ``ServiceNamespace``: +``WatchKube`` provider [#f3]_ +----------------------------- +.. _Kubernetes API: https://kubernetes.io/docs/reference/using-api/ +.. _watch requests: https://kubernetes.io/docs/reference/using-api/api-concepts/#efficient-detection-of-changes + +With this configuration, `Kubernetes API`_ "`watch requests`_" are used to fetch service configuration. +Essentially, it establishes one streamed HTTP connection with the `Kubernetes API`_ per downstream service. +Changes streamed through this connection will be used to update the list of available endpoints. .. code-block:: json - "Routes": [ - { - "ServiceName": "my-service", - "ServiceNamespace": "my-namespace" - } - ] + "ServiceDiscoveryProvider": { + "Namespace": "dev", + "Type": "WatchKube" + } + +The provider has an implicit configuration for fine-tuned watching, which are available and can only be initialized in C# code. + +* ``WatchKube.FirstResultsFetchingTimeoutSeconds``: `This `_ is the default number of seconds to wait after Ocelot starts, following the provider's creation, to fetch the first result from the Kubernetes endpoint. :sup:`1` +* ``WatchKube.FailedSubscriptionRetrySeconds``: `This `__ is the default number of seconds to wait before scheduling the next retry for the subscription operation. :sup:`1` + +.. _break3: http://break.do + + **Note 1**: For both ``static int`` properties, the default value is 1 (one) second. The constraint ensures that the assigned value is greater than or equal to 1 (one). Therefore, the minimum value is 1 (one) second. + + **Note 2**: The ``WatchKube`` provider is specifically designed for high-load Ocelot vs. Kubernetes environments with high RPS ratios. + To better understand which type is suitable for your needs, we have added a table :ref:`k8s-comparing-providers`. + +.. _k8s-comparing-providers: + +Comparing providers +------------------- +This table explains the most important indicators that may influence Ocelot vs. Kubernetes deployment or DevOps strategy. +The evolution path of all providers follows: ``Kube`` -> ``PollKube`` -> ``WatchKube``, with ``WatchKube`` being the most advanced provider. + +.. list-table:: + :widths: 34 22 22 22 + :header-rows: 1 + + * - *Indicators \\ Providers* + - :ref:`Kube ` + - :ref:`PollKube ` + - :ref:`WatchKube ` + * - Extra latency + - One hop per route + - \- + - \- + * - Speed of response to endpoints changes + - High + - Low :sup:`1` + - High + * - Pressure on `Kubernetes API`_ + - High + - Low :sup:`1` + - Low + * - Ocelot load (estimated) :sup:`2` + - < 1000 RPS + - > 1000 RPS + - > 5000 RPS + * - Ocelot deployment :sup:`3` + - Single instance + - Multiple instances + - Cluster of instances + +.. _break4: http://break.do + + | :sup:`1` Depends on the ``PollingInterval`` option. + | :sup:`2` Please consider this a rough load estimation, as our team has not provided any tests or benchmarks. + | :sup:`3` The term "instance" refers to an Ocelot instance, not a Kubernetes one. .. _k8s-downstream-scheme-vs-port-names: -Downstream Scheme vs Port Names [#f3]_ +Downstream Scheme vs Port Names [#f4]_ -------------------------------------- Kubernetes configuration permits the definition of multiple ports with names for each address of an endpoint subset. @@ -286,7 +357,7 @@ you must define ``DownstreamScheme`` to enable the provider to recognize the des } ] -.. _break3: http://break.do +.. _break5: http://break.do **Note**: In the absence of a specified ``DownstreamScheme`` (which is the default behavior), the ``Kube`` provider will select **the first available port** from the ``EndpointSubsetV1.Ports`` collection. Consequently, if the port name is not designated, the default downstream scheme utilized will be ``http``. @@ -295,13 +366,16 @@ you must define ``DownstreamScheme`` to enable the provider to recognize the des .. [#f1] The :doc:`../features/kubernetes` feature was requested as part of issue `345`_ to add support for `Kubernetes `_ :doc:`../features/servicediscovery` provider, and released in version `13.4.1`_ .. [#f2] The :ref:`k8s-addkubernetes-action-method` was requested as part of issue `2255`_ (PR `2257`_), and released in version `24.0`_ -.. [#f3] The :ref:`k8s-downstream-scheme-vs-port-names` feature was requested as part of issue `1967`_ and released in version `23.3`_ +.. [#f3] The :ref:`k8s-watchkube-provider` was discussed in thread `2168`_ and released in version `24.1`_ +.. [#f4] The :ref:`k8s-downstream-scheme-vs-port-names` feature was requested as part of issue `1967`_ and released in version `23.3`_ .. _345: https://github.com/ThreeMammals/Ocelot/issues/345 .. _1134: https://github.com/ThreeMammals/Ocelot/pull/1134 .. _1967: https://github.com/ThreeMammals/Ocelot/issues/1967 +.. _2168: https://github.com/ThreeMammals/Ocelot/discussions/2168 .. _2255: https://github.com/ThreeMammals/Ocelot/issues/2255 .. _2257: https://github.com/ThreeMammals/Ocelot/pull/2257 .. _13.4.1: https://github.com/ThreeMammals/Ocelot/releases/tag/13.4.1 .. _23.3: https://github.com/ThreeMammals/Ocelot/releases/tag/23.3.0 .. _24.0: https://github.com/ThreeMammals/Ocelot/releases/tag/24.0.0 +.. _24.1: https://github.com/ThreeMammals/Ocelot/releases/tag/24.1.0 diff --git a/src/Ocelot.Provider.Kubernetes/EndPointClientV1.cs b/src/Ocelot.Provider.Kubernetes/EndPointClientV1.cs index 40c5fcff1..cf8126def 100644 --- a/src/Ocelot.Provider.Kubernetes/EndPointClientV1.cs +++ b/src/Ocelot.Provider.Kubernetes/EndPointClientV1.cs @@ -7,7 +7,11 @@ namespace Ocelot.Provider.Kubernetes; public class EndPointClientV1 : KubeResourceClient, IEndPointClient { - private static readonly HttpRequest Collection = KubeRequest.Create("api/v1/namespaces/{Namespace}/endpoints/{ServiceName}"); + private static readonly HttpRequest EndpointsRequest = + KubeRequest.Create("api/v1/namespaces/{Namespace}/endpoints/{ServiceName}"); + + private static readonly HttpRequest EndpointsWatchRequest = + KubeRequest.Create("api/v1/watch/namespaces/{Namespace}/endpoints/{ServiceName}"); public EndPointClientV1(IKubeApiClient client) : base(client) { @@ -15,19 +19,29 @@ public EndPointClientV1(IKubeApiClient client) : base(client) public Task GetAsync(string serviceName, string kubeNamespace = null, CancellationToken cancellationToken = default) { - if (string.IsNullOrEmpty(serviceName)) - { - throw new ArgumentNullException(nameof(serviceName)); - } + ArgumentException.ThrowIfNullOrEmpty(serviceName); - var request = Collection - .WithTemplateParameters(new - { - Namespace = kubeNamespace ?? KubeClient.DefaultNamespace, - ServiceName = serviceName, - }); + var request = EndpointsRequest.WithTemplateParameters(new + { + Namespace = kubeNamespace ?? KubeClient.DefaultNamespace, + ServiceName = serviceName, + }); return Http.GetAsync(request, cancellationToken) - .ReadContentAsObjectV1Async(operationDescription: $"get {nameof(EndpointsV1)}"); + .ReadContentAsObjectV1Async(operationDescription: $"{nameof(GetAsync)} {nameof(EndpointsV1)}"); + } + + public IObservable> Watch(string serviceName, string kubeNamespace, CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrEmpty(serviceName); + + var request = EndpointsWatchRequest.WithTemplateParameters(new + { + ServiceName = serviceName, + Namespace = kubeNamespace ?? KubeClient.DefaultNamespace, + }); + + return ObserveEvents(request, + $"{nameof(Watch)} {nameof(EndpointsV1)} for '{serviceName}' in the namespace '{kubeNamespace ?? KubeClient.DefaultNamespace}'"); } } diff --git a/src/Ocelot.Provider.Kubernetes/Interfaces/IEndPointClient.cs b/src/Ocelot.Provider.Kubernetes/Interfaces/IEndPointClient.cs index 10f79f8af..8e85df73f 100644 --- a/src/Ocelot.Provider.Kubernetes/Interfaces/IEndPointClient.cs +++ b/src/Ocelot.Provider.Kubernetes/Interfaces/IEndPointClient.cs @@ -6,4 +6,6 @@ namespace Ocelot.Provider.Kubernetes.Interfaces; public interface IEndPointClient : IKubeResourceClient { Task GetAsync(string serviceName, string kubeNamespace = null, CancellationToken cancellationToken = default); + + IObservable> Watch(string serviceName, string kubeNamespace = null, CancellationToken cancellationToken = default); } diff --git a/src/Ocelot.Provider.Kubernetes/Kube.cs b/src/Ocelot.Provider.Kubernetes/Kube.cs index bc1942ed2..bfa6cc3c8 100644 --- a/src/Ocelot.Provider.Kubernetes/Kube.cs +++ b/src/Ocelot.Provider.Kubernetes/Kube.cs @@ -56,7 +56,7 @@ private async Task GetEndpoint() try { return await _kubeApi - .ResourceClient(client => new EndPointClientV1(client)) + .EndpointsV1() .GetAsync(_configuration.KeyOfServiceInK8s, _configuration.KubeNamespace); } catch (KubeApiException ex) diff --git a/src/Ocelot.Provider.Kubernetes/KubeApiClientExtensions.cs b/src/Ocelot.Provider.Kubernetes/KubeApiClientExtensions.cs new file mode 100644 index 000000000..6afa49927 --- /dev/null +++ b/src/Ocelot.Provider.Kubernetes/KubeApiClientExtensions.cs @@ -0,0 +1,9 @@ +using Ocelot.Provider.Kubernetes.Interfaces; + +namespace Ocelot.Provider.Kubernetes; + +public static class KubeApiClientExtensions +{ + public static IEndPointClient EndpointsV1(this IKubeApiClient client) + => client.ResourceClient(x => new EndPointClientV1(x)); +} diff --git a/src/Ocelot.Provider.Kubernetes/KubernetesProviderFactory.cs b/src/Ocelot.Provider.Kubernetes/KubernetesProviderFactory.cs index 4203b77c6..88431f2c8 100644 --- a/src/Ocelot.Provider.Kubernetes/KubernetesProviderFactory.cs +++ b/src/Ocelot.Provider.Kubernetes/KubernetesProviderFactory.cs @@ -2,6 +2,7 @@ using Ocelot.Configuration; using Ocelot.Logging; using Ocelot.Provider.Kubernetes.Interfaces; +using System.Reactive.Concurrency; namespace Ocelot.Provider.Kubernetes; @@ -9,6 +10,7 @@ public static class KubernetesProviderFactory // TODO : IServiceDiscoveryProvide { /// String constant used for provider type definition. public const string PollKube = nameof(Kubernetes.PollKube); + public const string WatchKube = nameof(Kubernetes.WatchKube); public static ServiceDiscoveryFinderDelegate Get { get; } = CreateProvider; @@ -17,7 +19,6 @@ private static IServiceDiscoveryProvider CreateProvider(IServiceProvider provide var factory = provider.GetService(); var kubeClient = provider.GetService(); var serviceBuilder = provider.GetService(); - var configuration = new KubeRegistryConfiguration { KeyOfServiceInK8s = route.ServiceName, @@ -25,6 +26,11 @@ private static IServiceDiscoveryProvider CreateProvider(IServiceProvider provide Scheme = route.DownstreamScheme, }; + if (WatchKube.Equals(config.Type, StringComparison.OrdinalIgnoreCase)) + { + return new WatchKube(configuration, factory, kubeClient, serviceBuilder, Scheduler.Default); + } + var defaultK8sProvider = new Kube(configuration, factory, kubeClient, serviceBuilder); return PollKube.Equals(config.Type, StringComparison.OrdinalIgnoreCase) diff --git a/src/Ocelot.Provider.Kubernetes/ObservableExtensions.cs b/src/Ocelot.Provider.Kubernetes/ObservableExtensions.cs new file mode 100644 index 000000000..f4ddbad80 --- /dev/null +++ b/src/Ocelot.Provider.Kubernetes/ObservableExtensions.cs @@ -0,0 +1,19 @@ +using System.Reactive.Concurrency; +using System.Reactive.Linq; + +namespace Ocelot.Provider.Kubernetes; + +public static class ObservableExtensions +{ + public static IObservable RetryAfter(this IObservable source, TimeSpan dueTime, IScheduler scheduler) + => RepeatInfinite(source, dueTime, scheduler).Catch(); + + private static IEnumerable> RepeatInfinite(IObservable source, TimeSpan dueTime, IScheduler scheduler) + { + yield return source; + while (true) + { + yield return source.DelaySubscription(dueTime, scheduler); + } + } +} diff --git a/src/Ocelot.Provider.Kubernetes/Ocelot.Provider.Kubernetes.csproj b/src/Ocelot.Provider.Kubernetes/Ocelot.Provider.Kubernetes.csproj index 8e3c6c508..bd2a1e2f1 100644 --- a/src/Ocelot.Provider.Kubernetes/Ocelot.Provider.Kubernetes.csproj +++ b/src/Ocelot.Provider.Kubernetes/Ocelot.Provider.Kubernetes.csproj @@ -48,5 +48,6 @@ + diff --git a/src/Ocelot.Provider.Kubernetes/WatchKube.cs b/src/Ocelot.Provider.Kubernetes/WatchKube.cs new file mode 100644 index 000000000..0e123a848 --- /dev/null +++ b/src/Ocelot.Provider.Kubernetes/WatchKube.cs @@ -0,0 +1,109 @@ +using KubeClient.Models; +using Ocelot.Logging; +using Ocelot.Provider.Kubernetes.Interfaces; +using Ocelot.Values; +using System.Reactive.Concurrency; +using System.Reactive.Linq; + +namespace Ocelot.Provider.Kubernetes; + +public class WatchKube : IServiceDiscoveryProvider, IDisposable +{ + /// The default number of seconds to wait before scheduling the next retry for the subscription operation. + /// A positive integer that is greater than or equal to 1. + public static int FailedSubscriptionRetrySeconds + { + get => failedSubscriptionRetrySeconds; + set => failedSubscriptionRetrySeconds = value >= 1 ? value : 1; + } + + /// The default number of seconds to wait after Ocelot starts, following the provider's creation, to fetch the first result from the Kubernetes endpoint. + /// A positive integer that is greater than or equal to 1. + public static int FirstResultsFetchingTimeoutSeconds + { + get => firstResultsFetchingTimeoutSeconds; + set => firstResultsFetchingTimeoutSeconds = value >= 1 ? value : 1; + } + + private static int failedSubscriptionRetrySeconds = 1; + private static int firstResultsFetchingTimeoutSeconds = 1; + + private readonly KubeRegistryConfiguration _configuration; + private readonly IOcelotLogger _logger; + private readonly IKubeApiClient _kubeApi; + private readonly IKubeServiceBuilder _serviceBuilder; + private readonly IScheduler _scheduler; + + private readonly IDisposable _subscription; + private TaskCompletionSource _firstResultsCompletionSource; + + private List _services = new(); + + public WatchKube( + KubeRegistryConfiguration configuration, + IOcelotLoggerFactory factory, + IKubeApiClient kubeApi, + IKubeServiceBuilder serviceBuilder, + IScheduler scheduler) + { + _configuration = configuration; + _logger = factory.CreateLogger(); + _kubeApi = kubeApi; + _serviceBuilder = serviceBuilder; + _scheduler = scheduler; + + SetFirstResultsCompletedAfterDelay(); + _subscription = CreateSubscription(); + } + + public virtual async Task> GetAsync() + { + // Wait for first results fetching + await _firstResultsCompletionSource.Task; + if (_services.Count == 0) + { + _logger.LogWarning(() => GetMessage("Subscription to service endpoints gave no results!")); + } + + return _services; + } + + private void SetFirstResultsCompletedAfterDelay() + { + _firstResultsCompletionSource = new(); + Observable + .Timer(TimeSpan.FromSeconds(FirstResultsFetchingTimeoutSeconds), _scheduler) + .Subscribe(_ => _firstResultsCompletionSource.TrySetResult()); + } + + private void OnNext(IResourceEventV1 endpointEvent) + { + _services = endpointEvent.EventType switch + { + ResourceEventType.Deleted or ResourceEventType.Error => new(), + _ when (endpointEvent.Resource?.Subsets.Count ?? 0) == 0 => new(), + _ => _serviceBuilder.BuildServices(_configuration, endpointEvent.Resource).ToList(), + }; + _firstResultsCompletionSource.TrySetResult(); + } + + // Called only when subscription canceled in Dispose + private void OnCompleted() => _logger.LogInformation(() => GetMessage("Subscription to service endpoints completed")); + private void OnException(Exception ex) => _logger.LogError(() => GetMessage("Endpoints subscription error occured."), ex); + + private IDisposable CreateSubscription() => _kubeApi + .EndpointsV1() + .Watch(_configuration.KeyOfServiceInK8s, _configuration.KubeNamespace) + .Do(_ => { }, OnException) + .RetryAfter(TimeSpan.FromSeconds(FailedSubscriptionRetrySeconds), _scheduler) + .Subscribe(OnNext, OnCompleted); + + private string GetMessage(string message) + => $"{nameof(WatchKube)} provider. Namespace:{_configuration.KubeNamespace}, Service:{_configuration.KeyOfServiceInK8s}; {message}"; + + public void Dispose() + { + _subscription.Dispose(); + GC.SuppressFinalize(this); + } +} diff --git a/test/Ocelot.AcceptanceTests/ConcurrentSteps.cs b/test/Ocelot.AcceptanceTests/ConcurrentSteps.cs index ff631d602..d43d8c29f 100644 --- a/test/Ocelot.AcceptanceTests/ConcurrentSteps.cs +++ b/test/Ocelot.AcceptanceTests/ConcurrentSteps.cs @@ -171,8 +171,19 @@ private async Task GetParallelResponse(string url, int threadIndex) public void ThenAllStatusCodesShouldBe(HttpStatusCode expected) => _responses.ShouldAllBe(response => response.Value.StatusCode == expected); + public void ThenAllResponseBodiesShouldBe(string expectedBody) - => _responses.ShouldAllBe(response => response.Value.Content.ReadAsStringAsync().Result == expectedBody); + { + foreach (var r in _responses) + { + var content = r.Value.Content.ReadAsStringAsync().Result; + content = content?.Contains(':') == true + ? content.Split(':')[1] // remove counter for body comparison + : "0"; + + content.ShouldBe(expectedBody); + } + } protected string CalledTimesMessage() => $"All values are [{string.Join(',', _counters)}]"; diff --git a/test/Ocelot.AcceptanceTests/ServiceDiscovery/KubernetesServiceDiscoveryTests.cs b/test/Ocelot.AcceptanceTests/ServiceDiscovery/KubernetesServiceDiscoveryTests.cs index 695043422..daf3dc3d3 100644 --- a/test/Ocelot.AcceptanceTests/ServiceDiscovery/KubernetesServiceDiscoveryTests.cs +++ b/test/Ocelot.AcceptanceTests/ServiceDiscovery/KubernetesServiceDiscoveryTests.cs @@ -38,8 +38,10 @@ public KubernetesServiceDiscoveryTests() }; } - [Fact] - public void ShouldReturnServicesFromK8s() + [Theory] + [InlineData(nameof(Kube))] + [InlineData(nameof(WatchKube))] + public void ShouldReturnServicesFromK8s(string discoveryType) { const string namespaces = nameof(KubernetesServiceDiscoveryTests); const string serviceName = nameof(ShouldReturnServicesFromK8s); @@ -49,12 +51,13 @@ public void ShouldReturnServicesFromK8s() var subsetV1 = GivenSubsetAddress(downstream); var endpoints = GivenEndpoints(subsetV1); var route = GivenRouteWithServiceName(namespaces); - var configuration = GivenKubeConfiguration(namespaces, route); + var configuration = GivenKubeConfiguration(namespaces, route, discoveryType); var downstreamResponse = serviceName; this.Given(x => GivenServiceInstanceIsRunning(downstreamUrl, downstreamResponse)) .And(x => x.GivenThereIsAFakeKubernetesProvider(endpoints, serviceName, namespaces)) .And(_ => GivenThereIsAConfiguration(configuration)) .And(_ => GivenOcelotIsRunning(WithKubernetes)) + .When(_ => GivenWatchReceivedEvent()) .When(_ => WhenIGetUrlOnTheApiGateway("/")) .Then(_ => ThenTheStatusCodeShouldBe(HttpStatusCode.OK)) .And(_ => ThenTheResponseBodyShouldBe($"1:{downstreamResponse}")) @@ -89,7 +92,7 @@ public void ShouldReturnServicesByPortNameAsDownstreamScheme(string downstreamSc route.DownstreamScheme = downstreamScheme; // !!! Warning !!! Select port by name as scheme route.UpstreamPathTemplate = "/api/example/{url}"; route.ServiceName = serviceName; // "example-web" - var configuration = GivenKubeConfiguration(namespaces, route); + var configuration = GivenKubeConfiguration(namespaces, route, nameof(Kube)); this.Given(x => GivenServiceInstanceIsRunning(downstreamUrl, nameof(ShouldReturnServicesByPortNameAsDownstreamScheme))) .And(x => x.GivenThereIsAFakeKubernetesProvider(endpoints, serviceName, namespaces)) @@ -157,9 +160,11 @@ public void ShouldHighlyLoadOnUnstableKubeProvider_WithRoundRobinLoadBalancing(i ThenServiceCountersShouldMatchLeasingCounters(_roundRobinAnalyzer, servicePorts, totalRequests); } - [Fact] + [Theory] + [InlineData(nameof(Kube))] + [InlineData(nameof(WatchKube))] [Trait("Feat", "2256")] - public void ShouldReturnServicesFromK8s_AddKubernetesWithNullConfigureOptions() + public void ShouldReturnServicesFromK8s_AddKubernetesWithNullConfigureOptions(string discoveryType) { const string namespaces = nameof(KubernetesServiceDiscoveryTests); const string serviceName = nameof(ShouldReturnServicesFromK8s_AddKubernetesWithNullConfigureOptions); @@ -169,12 +174,13 @@ public void ShouldReturnServicesFromK8s_AddKubernetesWithNullConfigureOptions() var subsetV1 = GivenSubsetAddress(downstream); var endpoints = GivenEndpoints(subsetV1); var route = GivenRouteWithServiceName(namespaces); - var configuration = GivenKubeConfiguration(namespaces, route, "txpc696iUhbVoudg164r93CxDTrKRVWG"); + var configuration = GivenKubeConfiguration(namespaces, route, discoveryType, "txpc696iUhbVoudg164r93CxDTrKRVWG"); var downstreamResponse = serviceName; this.Given(x => GivenServiceInstanceIsRunning(downstreamUrl, downstreamResponse)) .And(x => x.GivenThereIsAFakeKubernetesProvider(endpoints, serviceName, namespaces)) .And(_ => GivenThereIsAConfiguration(configuration)) .And(_ => GivenOcelotIsRunning(AddKubernetesWithNullConfigureOptions)) + .When(_ => GivenWatchReceivedEvent()) .When(_ => WhenIGetUrlOnTheApiGateway("/")) .Then(_ => ThenTheStatusCodeShouldBe(HttpStatusCode.OK)) .And(_ => ThenTheResponseBodyShouldBe($"1:{downstreamResponse}")) @@ -183,6 +189,58 @@ public void ShouldReturnServicesFromK8s_AddKubernetesWithNullConfigureOptions() .BDDfy(); } + [Fact] + [Trait("Feat", "2168")] + [Trait("PR", "2174")] // https://github.com/ThreeMammals/Ocelot/pull/2174 + public void ShouldReturnServicesFromK8s_OneWatchRequestUpdatesServicesInfo() + { + const string namespaces = nameof(KubernetesServiceDiscoveryTests); + const string serviceName = nameof(ShouldReturnServicesFromK8s_OneWatchRequestUpdatesServicesInfo); + (EndpointsV1 endpoints, string downstreamUrl) = GetServiceInstance(); + (EndpointsV1 updatedEndpoints, string updateDownstreamUrl) = GetServiceInstance(); + + ResourceEventV1[] events = + [ + new() { EventType = ResourceEventType.Added, Resource = endpoints }, + new() { EventType = ResourceEventType.Modified, Resource = updatedEndpoints } + ]; + + var route = GivenRouteWithServiceName(namespaces); + var configuration = GivenKubeConfiguration(namespaces, route, nameof(WatchKube)); + + var downstreamResponse = serviceName; + var updatedDownstreamResponse = "updated_content" + serviceName; + this.Given(x => GivenServiceInstanceIsRunning(downstreamUrl, downstreamResponse)) + .Given(x => GivenServiceInstanceIsRunning(updateDownstreamUrl, updatedDownstreamResponse)) + .And(x => x.GivenThereIsAFakeKubernetesProvider(events, serviceName, namespaces)) + .And(_ => GivenThereIsAConfiguration(configuration)) + .And(_ => GivenOcelotIsRunning(WithKubernetes)) + .When(_ => GivenWatchReceivedEvent()) + .When(_ => WhenIGetUrlOnTheApiGatewayConcurrently("/", 10)) + .Then(_ => ThenAllStatusCodesShouldBe(HttpStatusCode.OK)) + .Then(_ => ThenAllResponseBodiesShouldBe(downstreamResponse)) + .And(_ => ThenK8sShouldBeCalledExactly(1)) + .And(x => ThenAllServicesShouldHaveBeenCalledTimes(10)) + .When(_ => GivenWatchReceivedEvent()) + .Given(_ => GivenDelay(100)) + .When(_ => WhenIGetUrlOnTheApiGatewayConcurrently("/", 10)) + .Then(_ => ThenAllStatusCodesShouldBe(HttpStatusCode.OK)) + .Then(_ => ThenAllResponseBodiesShouldBe(updatedDownstreamResponse)) + .And(_ => ThenK8sShouldBeCalledExactly(1)) + .And(x => ThenAllServicesShouldHaveBeenCalledTimes(20)) + .BDDfy(); + + (EndpointsV1 Endpoints, string DownstreamUrl) GetServiceInstance() + { + var servicePort = PortFinder.GetRandomPort(); + var downstreamUrl = LoopbackLocalhostUrl(servicePort); + var downstream = new Uri(downstreamUrl); + var subset = GivenSubsetAddress(downstream); + var endpoints = GivenEndpoints(subset); + return (endpoints, downstreamUrl); + } + } + private void AddKubernetesWithNullConfigureOptions(IServiceCollection services) => services.AddOcelot().AddKubernetes(configureOptions: null); @@ -204,7 +262,7 @@ private void AddKubernetesWithNullConfigureOptions(IServiceCollection services) downstreams.ForEach(ds => GivenSubsetAddress(ds, subset)); var endpoints = GivenEndpoints(subset, serviceName); // totalServices service instances with different ports var route = GivenRouteWithServiceName(namespaces, serviceName, nameof(RoundRobinAnalyzer)); // !!! - var configuration = GivenKubeConfiguration(namespaces, route); + var configuration = GivenKubeConfiguration(namespaces, route, nameof(Kube)); GivenMultipleServiceInstancesAreRunning(downstreamUrls, downstreamResponses); GivenThereIsAConfiguration(configuration); GivenOcelotIsRunning(WithKubernetesAndRoundRobin); @@ -231,6 +289,11 @@ private void ThenTheTokenIs(string token) _receivedToken.ShouldBe(token); } + private void ThenK8sShouldBeCalledExactly(int totalRequests) + { + _k8sCounter.ShouldBe(totalRequests); + } + private EndpointsV1 GivenEndpoints(EndpointSubsetV1 subset, [CallerMemberName] string serviceName = "") { var e = new EndpointsV1() @@ -276,7 +339,7 @@ private FileRoute GivenRouteWithServiceName(string serviceNamespace, LoadBalancerOptions = new() { Type = loadBalancerType }, }; - private FileConfiguration GivenKubeConfiguration(string serviceNamespace, FileRoute route, string token = null) + private FileConfiguration GivenKubeConfiguration(string serviceNamespace, FileRoute route, string type, string token = null) { var u = new Uri(_kubernetesUrl); var configuration = GivenConfiguration(route); @@ -285,7 +348,7 @@ private FileConfiguration GivenKubeConfiguration(string serviceNamespace, FileRo Scheme = u.Scheme, Host = u.Host, Port = u.Port, - Type = nameof(Kube), + Type = type, PollingInterval = 0, Namespace = serviceNamespace, Token = token ?? "Test", @@ -337,12 +400,63 @@ private void GivenThereIsAFakeKubernetesProvider(EndpointsV1 endpoints, bool isS context.Response.Headers.Append("Content-Type", "application/json"); await context.Response.WriteAsync(json); } + + await GivenHandleWatchRequest(context, + [new() { EventType = ResourceEventType.Added, Resource = endpoints }], + namespaces, + serviceName); }); } + + private void GivenThereIsAFakeKubernetesProvider(ResourceEventV1[] events, + [CallerMemberName] string serviceName = nameof(KubernetesServiceDiscoveryTests), + string namespaces = nameof(KubernetesServiceDiscoveryTests)) + { + _k8sCounter = 0; + handler.GivenThereIsAServiceRunningOn(_kubernetesUrl, (c) => GivenHandleWatchRequest(c, events, namespaces, serviceName)); + } + + private void GivenWatchReceivedEvent() => _k8sWatchResetEvent.Set(); + + private static Task GivenDelay(int milliseconds) => Task.Delay(TimeSpan.FromMilliseconds(milliseconds)); + + private async Task GivenHandleWatchRequest(HttpContext context, + IEnumerable> events, + string namespaces, + string serviceName) + { + await Task.Delay(Random.Shared.Next(1, 10)); // emulate integration delay up to 10 milliseconds + + if (context.Request.Path.Value == $"/api/v1/watch/namespaces/{namespaces}/endpoints/{serviceName}") + { + _k8sCounter++; + + if (context.Request.Headers.TryGetValue("Authorization", out var values)) + { + _receivedToken = values.First(); + } + + context.Response.StatusCode = 200; + context.Response.Headers.Append("Content-Type", "application/json"); + + foreach (var @event in events) + { + _k8sWatchResetEvent.WaitOne(); + var json = JsonConvert.SerializeObject(@event, KubeResourceClient.SerializerSettings); + await using var sw = new StreamWriter(context.Response.Body); + await sw.WriteLineAsync(json); + await sw.FlushAsync(); + _k8sWatchResetEvent.Reset(); + } + + // keeping open connection like kube api will slow down tests + } + } private static ServiceDescriptor GetValidateScopesDescriptor() => ServiceDescriptor.Singleton>( new DefaultServiceProviderFactory(new() { ValidateScopes = true })); + private IOcelotBuilder AddKubernetes(IServiceCollection services) => services .Replace(GetValidateScopesDescriptor()) .AddOcelot().AddKubernetes(_kubeClientOptionsConfigure); @@ -355,6 +469,7 @@ private void WithKubernetesAndRoundRobin(IServiceCollection services) => AddKube private int _k8sCounter, _k8sServiceGeneration; private static readonly object K8sCounterLocker = new(); private RoundRobinAnalyzer _roundRobinAnalyzer; + private AutoResetEvent _k8sWatchResetEvent = new(false); private RoundRobinAnalyzer GetRoundRobinAnalyzer(DownstreamRoute route, IServiceDiscoveryProvider provider) { lock (K8sCounterLocker) diff --git a/test/Ocelot.UnitTests/Kubernetes/EndpointClientV1Tests.cs b/test/Ocelot.UnitTests/Kubernetes/EndpointClientV1Tests.cs new file mode 100644 index 000000000..4972521d9 --- /dev/null +++ b/test/Ocelot.UnitTests/Kubernetes/EndpointClientV1Tests.cs @@ -0,0 +1,130 @@ +using KubeClient; +using KubeClient.Http; +using KubeClient.Http.Formatters; +using KubeClient.Models; +using Microsoft.Extensions.Logging; +using Ocelot.Provider.Kubernetes; +using Ocelot.Provider.Kubernetes.Interfaces; +using System.Runtime.CompilerServices; + +namespace Ocelot.UnitTests.Kubernetes; + +[Trait("Feat", "2168")] +[Trait("PR", "2174")] // https://github.com/ThreeMammals/Ocelot/pull/2174 +public class EndpointClientV1Tests +{ + private readonly EndPointClientV1 _endpointClient; + private readonly Mock _kubeApiClient = new(); + + public EndpointClientV1Tests() + { + var loggerFactory = new Mock(); + loggerFactory.Setup(x => x.CreateLogger(It.IsAny())) + .Returns(new Mock().Object); + _kubeApiClient.Setup(x => x.LoggerFactory) + .Returns(loggerFactory.Object); + _endpointClient = new EndPointClientV1(_kubeApiClient.Object); + _kubeApiClient.Setup(x => x.ResourceClient(It.IsAny>())) + .Returns(_endpointClient); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + public async Task GetAsync_WhenServiceIsNullOrEmpty_ThrowsArgumentException(string serviceName) + { + // Act + var watchCall = () => _endpointClient.GetAsync(serviceName, null, CancellationToken.None); + + // Assert + var e = await watchCall.ShouldThrowAsync(); + e.ParamName.ShouldBe(nameof(serviceName)); + } + + [Theory] + [InlineData(null)] + [InlineData("test-namespace")] + public async Task GetAsync_KubeNamespaceChanges_HappyPath(string kubeNamespace) + { + // Arrange + using var client = new FakeHttpClient() + { + BaseAddress = new UriBuilder(Uri.UriSchemeHttp, "localhost", 1234).Uri, + }; + _kubeApiClient.SetupGet(x => x.Http).Returns(client); + _kubeApiClient.SetupGet(x => x.DefaultNamespace).Returns(nameof(EndpointClientV1Tests)); + + // Act + var endpoints = await _endpointClient.GetAsync("service-XYZ", kubeNamespace, CancellationToken.None); + + // Assert + Assert.NotNull(endpoints); + Assert.Equal(nameof(GetAsync_KubeNamespaceChanges_HappyPath), endpoints.Kind); + var url = client.Request.RequestUri.AbsoluteUri; + Assert.Contains(kubeNamespace ?? nameof(EndpointClientV1Tests), url); + Assert.Contains("service-XYZ", url); + client.Request.Options.TryGetValue(new("KubeClient.Http.Request"), out HttpRequest request); + Assert.NotNull(request?.TemplateParameters); + Assert.True(request.TemplateParameters.ContainsKey("Namespace")); + Assert.True(request.TemplateParameters.ContainsKey("ServiceName")); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + public void Watch_WhenServiceIsNullOrEmpty_ThrowsArgumentException(string serviceName) + { + // Act + var watchCall = () => _endpointClient.Watch(serviceName, null, CancellationToken.None); + + // Assert + watchCall.ShouldThrow().ParamName.ShouldBe(nameof(serviceName)); + } + + [Fact] + public void Watch_ProvidesObservable() + { + // Act + var observable = _endpointClient.Watch("some-service", null, CancellationToken.None); + + // Assert + observable.ShouldNotBeNull(); + } +} + +internal class FakeHttpClient : HttpClient, IDisposable +{ + private readonly Mock formatters = new(); + private readonly Mock formatter = new(); + private readonly List disposables = new(); + public FakeHttpClient([CallerMemberName] string testName = null) + { + formatter.Setup(x => x.ReadAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(() => new EndpointsV1() { Kind = testName }); + formatters.SetupGet(x => x.Count).Returns(1); + formatters.Setup(x => x.FindInputFormatter(It.IsAny())) + .Returns(formatter.Object); + } + + public new void Dispose() + { + disposables.ForEach(d => d.Dispose()); + base.Dispose(); + } + + public HttpRequestMessage Request { get; private set; } + public override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + Request = request; + HttpResponseMessage response = new() + { + StatusCode = HttpStatusCode.OK, + RequestMessage = new(), + }; + response.RequestMessage.Properties.Add(MessageProperties.ContentFormatters, formatters.Object); + response.Content = new StringContent("Hello from " + nameof(FakeHttpClient)); + disposables.Add(response); + disposables.Add(response.Content); + return Task.FromResult(response); + } +} diff --git a/test/Ocelot.UnitTests/Kubernetes/KubeProviderFactoryTests.cs b/test/Ocelot.UnitTests/Kubernetes/KubeProviderFactoryTests.cs deleted file mode 100644 index 0627e7ba6..000000000 --- a/test/Ocelot.UnitTests/Kubernetes/KubeProviderFactoryTests.cs +++ /dev/null @@ -1,35 +0,0 @@ -using Microsoft.Extensions.DependencyInjection; -using Moq; -using Ocelot.Configuration; -using Ocelot.Logging; -using Ocelot.Provider.Kubernetes; -using Shouldly; -using System; -using Xunit; - -namespace Ocelot.UnitTests.Kubernetes -{ - public class KubeProviderFactoryTests - { - private readonly IServiceProvider _provider; - - public KubeProviderFactoryTests() - { - var services = new ServiceCollection(); - var loggerFactory = new Mock(); - var logger = new Mock(); - loggerFactory.Setup(x => x.CreateLogger()).Returns(logger.Object); - var kubeFactory = new Mock(); - services.AddSingleton(kubeFactory.Object); - services.AddSingleton(loggerFactory.Object); - _provider = services.BuildServiceProvider(); - } - - [Fact] - public void should_return_KubeServiceDiscoveryProvider() - { - var provider = KubernetesProviderFactory.Get(_provider, new ServiceProviderConfiguration("kube", "localhost", 443, "", "", 1,"dev"), ""); - provider.ShouldBeOfType(); - } - } -} diff --git a/test/Ocelot.UnitTests/Kubernetes/KubernetesProviderFactoryTests.cs b/test/Ocelot.UnitTests/Kubernetes/KubernetesProviderFactoryTests.cs index 7ebea3f49..ab04e4a5c 100644 --- a/test/Ocelot.UnitTests/Kubernetes/KubernetesProviderFactoryTests.cs +++ b/test/Ocelot.UnitTests/Kubernetes/KubernetesProviderFactoryTests.cs @@ -1,4 +1,5 @@ using KubeClient; +using KubeClient.Models; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; @@ -44,13 +45,18 @@ public KubernetesProviderFactoryTests() [Trait("Bug", "977")] [InlineData(typeof(Kube))] [InlineData(typeof(PollKube))] + [InlineData(typeof(WatchKube))] public void CreateProvider_ClientHasOriginalLifetimeWithEnabledScopesValidation_ShouldResolveProvider(Type providerType) { // Arrange _builder.AddKubernetes(); + var endpointClient = new Mock(); + endpointClient.Setup(x => x.Watch(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(Mock.Of>>()); + var kubeClient = new Mock(); kubeClient.Setup(x => x.ResourceClient(It.IsAny>())) - .Returns(Mock.Of()); + .Returns(endpointClient.Object); var descriptor = _builder.Services.First(x => x.ServiceType == typeof(IKubeApiClient)); _builder.Services.Replace(ServiceDescriptor.Describe(descriptor.ServiceType, _ => kubeClient.Object, descriptor.Lifetime)); @@ -65,6 +71,7 @@ public void CreateProvider_ClientHasOriginalLifetimeWithEnabledScopesValidation_ [Trait("Bug", "977")] [InlineData(nameof(Kube))] [InlineData(nameof(PollKube))] + [InlineData(nameof(WatchKube))] public void CreateProvider_ClientHasScopedLifetimeWithEnabledScopesValidation_ShouldFailToResolve(string providerType) { // Arrange diff --git a/test/Ocelot.UnitTests/Kubernetes/ObservableExtensionsTests.cs b/test/Ocelot.UnitTests/Kubernetes/ObservableExtensionsTests.cs new file mode 100644 index 000000000..74ae70d60 --- /dev/null +++ b/test/Ocelot.UnitTests/Kubernetes/ObservableExtensionsTests.cs @@ -0,0 +1,53 @@ +using Microsoft.Reactive.Testing; +using Ocelot.Provider.Kubernetes; +using System.Reactive.Disposables; +using System.Reactive.Linq; + +namespace Ocelot.UnitTests.Kubernetes; + +[Trait("Feat", "2168")] +[Trait("PR", "2174")] // https://github.com/ThreeMammals/Ocelot/pull/2174 +public class ObservableExtensionsTests +{ + private readonly TestScheduler _testScheduler = new(); + + [Fact] + public async Task RetryAfter_ExceptionThrown_RetriesInfiniteWithDelay() + { + // Arrange + var errorsToThrow = Random.Shared.Next(10, 1000); + var errorsCounter = 0; + var expectedResult = 123; + var delaySeconds = TimeSpan.FromSeconds(3); + var observable = Observable.Create(observer => + { + if (errorsCounter < errorsToThrow) + { + errorsCounter++; + throw new Exception("Need to catch and retry"); + } + + observer.OnNext(expectedResult); + return Disposable.Empty; + }); + + // Act + using var cts = new CancellationTokenSource(); + _ = Task.Run(() => + { + // have to spin in separate thread because it is used after first subscription and stops after first Exception + while (!cts.Token.IsCancellationRequested) + { + _testScheduler.Start(); + } + }); + + var result = await observable.RetryAfter(delaySeconds, _testScheduler).FirstAsync(); + await cts.CancelAsync(); + + // Assert + result.ShouldBe(expectedResult); + errorsCounter.ShouldBe(errorsToThrow); + _testScheduler.Clock.ShouldBe(delaySeconds.Ticks * errorsToThrow); + } +} diff --git a/test/Ocelot.UnitTests/Kubernetes/WatchKubeTests.cs b/test/Ocelot.UnitTests/Kubernetes/WatchKubeTests.cs new file mode 100644 index 000000000..8ff5e5e03 --- /dev/null +++ b/test/Ocelot.UnitTests/Kubernetes/WatchKubeTests.cs @@ -0,0 +1,230 @@ +using KubeClient; +using KubeClient.Models; +using Microsoft.Extensions.Logging; +using Microsoft.Reactive.Testing; +using Ocelot.Infrastructure.RequestData; +using Ocelot.Logging; +using Ocelot.Provider.Kubernetes; +using Ocelot.Provider.Kubernetes.Interfaces; +using Ocelot.Responses; +using Ocelot.Values; +using System.Linq.Expressions; +using System.Reactive.Linq; + +namespace Ocelot.UnitTests.Kubernetes; + +[Trait("Feat", "2168")] +[Trait("PR", "2174")] // https://github.com/ThreeMammals/Ocelot/pull/2174 +public class WatchKubeTests +{ + private readonly Mock _loggerFactory = new(); + private readonly Mock _kubeApiClient = new(); + private readonly Mock _endpointClient = new(); + private readonly Mock _kubeServiceBuilder = new(); + private readonly TestScheduler _testScheduler = new(); + private readonly KubeRegistryConfiguration _config = new() + { + KubeNamespace = "dummy-namespace", KeyOfServiceInK8s = "dummy-service", + }; + private readonly OcelotLogger _ocLogger; + private readonly Mock _logger = new(); + private readonly Mock _dataRepository = new(); + private readonly Expression>>> _watch; + + public WatchKubeTests() + { + _logger.Setup(x => x.IsEnabled(It.IsAny())) + .Returns(true); + _logger.Setup(x => x.Log(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())) + .Verifiable(); + _dataRepository.Setup(x => x.Get(It.IsAny())) + .Returns((Response)null); + _ocLogger = new(_logger.Object, _dataRepository.Object); + _loggerFactory.Setup(x => x.CreateLogger()) + .Returns(_ocLogger); + _kubeApiClient.Setup(x => x.ResourceClient(It.IsAny>())) + .Returns(_endpointClient.Object); + _kubeServiceBuilder.Setup(x => x.BuildServices(It.IsAny(), It.IsAny())) + .Returns((KubeRegistryConfiguration config, EndpointsV1 endpoints) => + { + return endpoints.Subsets.Select((x, i) => new Service( + config.KeyOfServiceInK8s, + new ServiceHostAndPort(x.Addresses[i].Hostname, x.Ports[i].Port!.Value), + i.ToString(), + endpoints.ApiVersion, + Enumerable.Empty())); + }); + _watch = x => x.Watch(It.Is(s => s == _config.KeyOfServiceInK8s), It.IsAny(), It.IsAny()); + } + + [Theory] + [InlineData(ResourceEventType.Added, 1)] + [InlineData(ResourceEventType.Modified, 1)] + [InlineData(ResourceEventType.Bookmark, 1)] + [InlineData(ResourceEventType.Error, 0)] + [InlineData(ResourceEventType.Deleted, 0)] + public async Task GetAsync_EndpointsEventObserved_ServicesReturned(ResourceEventType eventType, int expectedServicesCount) + { + // Arrange + var eventDelay = TimeSpan.FromMilliseconds(Random.Shared.Next(1, (WatchKube.FirstResultsFetchingTimeoutSeconds * 1000) - 1)); + var endpointsObservable = CreateOneEvent(eventType).ToObservable().Delay(eventDelay, _testScheduler); + _endpointClient.Setup(_watch).Returns(endpointsObservable); + + // Act + var watchKube = CreateWatchKube(); + _testScheduler.AdvanceBy(eventDelay.Ticks); + var services = await watchKube.GetAsync(); + + // Assert + services.Count.ShouldBe(expectedServicesCount); + } + + [Fact] + public async Task GetAsync_NoEventsAfterTimeout_EmptyServicesReturned() + { + // Arrange + _endpointClient.Setup(_watch) + .Returns(Observable.Create>(_ => Mock.Of())); + + // Act + var watchKube = CreateWatchKube(); + _testScheduler.Start(); + var services = await watchKube.GetAsync(); + + // Assert + services.ShouldBeEmpty(); + _testScheduler.Clock.ShouldBe(TimeSpan.FromSeconds(WatchKube.FirstResultsFetchingTimeoutSeconds).Ticks); + _logger.Verify( + x => x.Log(LogLevel.Warning, It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>()), + Times.Once()); + } + + [Fact] + public async Task GetAsync_WatchFailed_RetriedAfterDelay() + { + // Arrange + var subscriptionAttempts = 0; + var observable = Observable.Create>(observer => + { + if (subscriptionAttempts == 0) + { + observer.OnError(new HttpRequestException("Error occured in first watch request")); + } + else + { + observer.OnNext(CreateOneEvent(ResourceEventType.Added).First()); + } + + subscriptionAttempts++; + return Mock.Of(); + }); + _endpointClient.Setup(_watch).Returns(observable); + + // Act + var watchKube = CreateWatchKube(); + _testScheduler.Start(); + var services = await watchKube.GetAsync(); + + // Assert + services.Count.ShouldBe(1); + subscriptionAttempts.ShouldBe(2); + _testScheduler.Clock.ShouldBe(TimeSpan.FromSeconds(WatchKube.FailedSubscriptionRetrySeconds).Ticks); + _logger.Verify( + x => x.Log(LogLevel.Error, It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>()), + Times.Once()); + } + + [Fact] + public async Task Dispose_OnSubscriptionCancellation_LogsInformation() + { + // Arrange + var observable = Observable.Create>(observer => + { + observer.OnCompleted(); + return Mock.Of(); + }); + _endpointClient.Setup(_watch).Returns(observable); + + // Act + var watchKube = CreateWatchKube(); + _testScheduler.Start(); + var services = await watchKube.GetAsync(); + watchKube.Dispose(); + + // Assert + services.ShouldBeEmpty(); + _testScheduler.Clock.ShouldBe(TimeSpan.FromSeconds(WatchKube.FirstResultsFetchingTimeoutSeconds).Ticks); + _logger.Verify( + x => x.Log(LogLevel.Information, It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>()), + Times.Once()); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task GetAsync_EndpointsEventObserved_NoServices(bool branch1) + { + // Arrange + var eventDelay = TimeSpan.FromMilliseconds(Random.Shared.Next(1, (WatchKube.FirstResultsFetchingTimeoutSeconds * 1000) - 1)); + EndpointsV1 endpoints = null; + if (branch1) + { + endpoints = new EndpointsV1 + { + Kind = "endpoint", ApiVersion = "1.0", + Metadata = new ObjectMetaV1 { Name = _config.KeyOfServiceInK8s, Namespace = _config.KubeNamespace, }, + }; + endpoints.Subsets.Clear(); + } + + var resourceEvent = new ResourceEventV1 { EventType = ResourceEventType.Bookmark, Resource = endpoints, }; + var events = new ResourceEventV1[] { resourceEvent }; + var endpointsObservable = events.ToObservable().Delay(eventDelay, _testScheduler); + _endpointClient.Setup(_watch).Returns(endpointsObservable); + + // Act + var watchKube = CreateWatchKube(); + _testScheduler.AdvanceBy(eventDelay.Ticks); + var services = await watchKube.GetAsync(); + + // Assert + services.ShouldBeEmpty(); + } + + [Theory] + [InlineData(-1, 1)] + [InlineData(0, 1)] + [InlineData(1, 1)] + [InlineData(3, 3)] + public void StaticProperties_Setter_ShouldBeGreaterThanOrEqualToOne(int value, int expected) + { + WatchKube.FailedSubscriptionRetrySeconds = value; + Assert.Equal(expected, WatchKube.FailedSubscriptionRetrySeconds); + + WatchKube.FirstResultsFetchingTimeoutSeconds = value; + Assert.Equal(expected, WatchKube.FirstResultsFetchingTimeoutSeconds); + } + + private WatchKube CreateWatchKube() => new(_config, _loggerFactory.Object, _kubeApiClient.Object, _kubeServiceBuilder.Object, _testScheduler); + + private IResourceEventV1[] CreateOneEvent(ResourceEventType eventType) + { + var resourceEvent = new ResourceEventV1 { EventType = eventType, Resource = CreateEndpoints(), }; + return [resourceEvent]; + } + + private EndpointsV1 CreateEndpoints() + { + var endpoints = new EndpointsV1 + { + Kind = "endpoint", + ApiVersion = "1.0", + Metadata = new ObjectMetaV1 { Name = _config.KeyOfServiceInK8s, Namespace = _config.KubeNamespace, }, + }; + var subset = new EndpointSubsetV1(); + subset.Addresses.Add(new EndpointAddressV1 { Ip = "127.0.0.1", Hostname = "localhost" }); + subset.Ports.Add(new EndpointPortV1 { Port = 80 }); + endpoints.Subsets.Add(subset); + return endpoints; + } +} diff --git a/test/Ocelot.UnitTests/Ocelot.UnitTests.csproj b/test/Ocelot.UnitTests/Ocelot.UnitTests.csproj index 7107e27d0..4d86c9e42 100644 --- a/test/Ocelot.UnitTests/Ocelot.UnitTests.csproj +++ b/test/Ocelot.UnitTests/Ocelot.UnitTests.csproj @@ -22,9 +22,6 @@ full True - - - @@ -79,6 +76,7 @@ +