diff --git a/docs/features/kubernetes.rst b/docs/features/kubernetes.rst index d7d9b71a7..3c155542c 100644 --- a/docs/features/kubernetes.rst +++ b/docs/features/kubernetes.rst @@ -36,7 +36,7 @@ The first thing you need to do is install the `package`_ that provides |kubernet Install-Package Ocelot.Provider.Kubernetes ``AddKubernetes(bool)`` method -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +------------------------------ .. code-block:: csharp :emphasize-lines: 3 @@ -84,19 +84,40 @@ If you have services deployed in Kubernetes, you will normally use the naming se }; builder.Services .AddOptions() - .Configure(configureKubeClient); // mannual binding options via IOptions + .Configure(configureKubeClient); // manual binding options via IOptions builder.Services .AddOcelot(builder.Configuration) .AddKubernetes(false); // don't use pod service account, and IOptions is reused + .. _break: http://break.do + + **Note**, this could also be written like this (shortened version): + + .. code-block:: csharp + :emphasize-lines: 2, 10 + + builder.Services + .AddKubeClientOptions(opts => + { + opts.ApiEndPoint = new UriBuilder("https", "my-host", 443).Uri; + opts.AuthStrategy = KubeAuthStrategy.BearerToken; + opts.AccessToken = "my-token"; + opts.AllowInsecure = true; + }) + .AddOcelot(builder.Configuration) + .AddKubernetes(false); // don't use pod service account, and client options provided via AddKubeClientOptions + Finally, it creates the `KubeClient`_ from your options. - **Note**: For understanding the ``IOptions`` interface, please refer to the Microsoft Learn documentation: `Options pattern in .NET `_. + **Note 1**: For understanding the ``IOptions`` interface, please refer to the Microsoft Learn documentation: `Options pattern in .NET `_. + + **Note 2**: Please consider this Case 2 as an example of manual setup when you **do not** use a pod service account. + We recommend using our official extension method, which receives an ``Action`` argument with your options: refer to the :ref:`k8s-addkubernetes-action-method` below. .. _k8s-addkubernetes-action-method: ``AddKubernetes(Action)`` method [#f2]_ -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +---------------------------------------------------------- .. code-block:: csharp :emphasize-lines: 3 diff --git a/samples/Kubernetes/ApiGateway/Program.cs b/samples/Kubernetes/ApiGateway/Program.cs index ba9c98f4b..988727900 100644 --- a/samples/Kubernetes/ApiGateway/Program.cs +++ b/samples/Kubernetes/ApiGateway/Program.cs @@ -10,7 +10,7 @@ .SetBasePath(builder.Environment.ContentRootPath) .AddOcelot(); -goto Case4; // Your case should be selected here!!! +goto Case5; // Your case should be selected here!!! // Link: https://github.com/ThreeMammals/Ocelot/blob/develop/docs/features/kubernetes.rst#addkubernetes-bool-method Case1: // Use a pod service account @@ -36,7 +36,7 @@ goto Start; // Link: https://github.com/ThreeMammals/Ocelot/blob/develop/docs/features/kubernetes.rst#addkubernetes-action-kubeclientoptions-method -Case3: // Use global ServiceDiscoveryProvider json-options +Case3: // Don't use a pod service account, manually bind options, ignore global ServiceDiscoveryProvider json-options Action myOptions = opts => { opts.ApiEndPoint = new UriBuilder(Uri.UriSchemeHttps, "my-host", 443).Uri; @@ -49,8 +49,21 @@ .AddKubernetes(myOptions); // configure options with action, without optional args goto Start; +Case4: // Don't use a pod service account, manually bind options, ignore global ServiceDiscoveryProvider json-options +builder.Services + .AddKubeClientOptions(opts => + { + opts.ApiEndPoint = new UriBuilder("https", "my-host", 443).Uri; + opts.AuthStrategy = KubeAuthStrategy.BearerToken; + opts.AccessToken = "my-token"; + opts.AllowInsecure = true; + }) + .AddOcelot(builder.Configuration) + .AddKubernetes(false); // don't use pod service account, and client options provided via AddKubeClientOptions +goto Start; + // Link: https://github.com/ThreeMammals/Ocelot/blob/develop/docs/features/kubernetes.rst#addkubernetes-action-kubeclientoptions-method -Case4: // Use global ServiceDiscoveryProvider json-options +Case5: // Use global ServiceDiscoveryProvider json-options Action? none = null; builder.Services .AddOcelot(builder.Configuration) diff --git a/src/Ocelot.Provider.Kubernetes/EndPointClientV1.cs b/src/Ocelot.Provider.Kubernetes/EndPointClientV1.cs index 6b81693d4..40c5fcff1 100644 --- a/src/Ocelot.Provider.Kubernetes/EndPointClientV1.cs +++ b/src/Ocelot.Provider.Kubernetes/EndPointClientV1.cs @@ -1,4 +1,4 @@ -using HTTPlease; +using KubeClient.Http; using KubeClient.Models; using KubeClient.ResourceClients; using Ocelot.Provider.Kubernetes.Interfaces; @@ -7,31 +7,27 @@ namespace Ocelot.Provider.Kubernetes; public class EndPointClientV1 : KubeResourceClient, IEndPointClient { - private readonly HttpRequest _collection; + private static readonly HttpRequest Collection = KubeRequest.Create("api/v1/namespaces/{Namespace}/endpoints/{ServiceName}"); public EndPointClientV1(IKubeApiClient client) : base(client) { - _collection = KubeRequest.Create("api/v1/namespaces/{Namespace}/endpoints/{ServiceName}"); } - public async Task GetAsync(string serviceName, string kubeNamespace = null, CancellationToken cancellationToken = default) + public Task GetAsync(string serviceName, string kubeNamespace = null, CancellationToken cancellationToken = default) { if (string.IsNullOrEmpty(serviceName)) { throw new ArgumentNullException(nameof(serviceName)); } - var request = _collection + var request = Collection .WithTemplateParameters(new { Namespace = kubeNamespace ?? KubeClient.DefaultNamespace, ServiceName = serviceName, }); - var response = await Http.GetAsync(request, cancellationToken); - - return response.IsSuccessStatusCode - ? await response.ReadContentAsAsync() - : null; + return Http.GetAsync(request, cancellationToken) + .ReadContentAsObjectV1Async(operationDescription: $"get {nameof(EndpointsV1)}"); } } diff --git a/src/Ocelot.Provider.Kubernetes/Kube.cs b/src/Ocelot.Provider.Kubernetes/Kube.cs index 6ae36bb71..bc1942ed2 100644 --- a/src/Ocelot.Provider.Kubernetes/Kube.cs +++ b/src/Ocelot.Provider.Kubernetes/Kube.cs @@ -15,6 +15,8 @@ namespace Ocelot.Provider.Kubernetes; /// public class Kube : IServiceDiscoveryProvider { + private static readonly (string ResourceKind, string ResourceApiVersion) EndPointsKubeKind = KubeObjectV1.GetKubeKind(); + private readonly KubeRegistryConfiguration _configuration; private readonly IOcelotLogger _logger; private readonly IKubeApiClient _kubeApi; @@ -46,9 +48,44 @@ public virtual async Task> GetAsync() .ToList(); } - private Task GetEndpoint() => _kubeApi - .ResourceClient(client => new EndPointClientV1(client)) - .GetAsync(_configuration.KeyOfServiceInK8s, _configuration.KubeNamespace); + private string Message(string details) + => $"Failed to retrieve {EndPointsKubeKind.ResourceApiVersion}/{EndPointsKubeKind.ResourceKind} '{_configuration.KeyOfServiceInK8s}' in namespace '{_configuration.KubeNamespace}': {details}"; + + private async Task GetEndpoint() + { + try + { + return await _kubeApi + .ResourceClient(client => new EndPointClientV1(client)) + .GetAsync(_configuration.KeyOfServiceInK8s, _configuration.KubeNamespace); + } + catch (KubeApiException ex) + { + string Msg() + { + StatusV1 status = ex.Status; + string httpStatusCode = "-"; // Unknown + if (ex.InnerException is HttpRequestException e) + { + httpStatusCode = e.StatusCode.ToString(); + } + + return Message($"(HTTP.{httpStatusCode}/{status.Status}/{status.Reason}): {status.Message}"); + } + + _logger.LogError(Msg, ex); + } + catch (HttpRequestException ex) + { + _logger.LogError(() => Message($"({ex.HttpRequestError}/HTTP.{ex.StatusCode})."), ex); + } + catch (Exception unexpected) + { + _logger.LogError(() => Message($"(an unexpected ex occurred)."), unexpected); + } + + return null; + } private bool CheckErroneousState(EndpointsV1 endpoint) => (endpoint?.Subsets?.Count ?? 0) == 0; // null or count is zero diff --git a/src/Ocelot.Provider.Kubernetes/Ocelot.Provider.Kubernetes.csproj b/src/Ocelot.Provider.Kubernetes/Ocelot.Provider.Kubernetes.csproj index 711ff0e80..d2ad0acf7 100644 --- a/src/Ocelot.Provider.Kubernetes/Ocelot.Provider.Kubernetes.csproj +++ b/src/Ocelot.Provider.Kubernetes/Ocelot.Provider.Kubernetes.csproj @@ -25,8 +25,8 @@ 1591 - - + + all diff --git a/test/Ocelot.AcceptanceTests/Ocelot.AcceptanceTests.csproj b/test/Ocelot.AcceptanceTests/Ocelot.AcceptanceTests.csproj index 2e88aaaee..ec8d0e8b1 100644 --- a/test/Ocelot.AcceptanceTests/Ocelot.AcceptanceTests.csproj +++ b/test/Ocelot.AcceptanceTests/Ocelot.AcceptanceTests.csproj @@ -73,14 +73,14 @@ all - - - - - - - - + + + + + + + + @@ -88,6 +88,6 @@ - + diff --git a/test/Ocelot.AcceptanceTests/ServiceDiscovery/KubernetesServiceDiscoveryTests.cs b/test/Ocelot.AcceptanceTests/ServiceDiscovery/KubernetesServiceDiscoveryTests.cs index 9b6a5c6da..fa3841ec8 100644 --- a/test/Ocelot.AcceptanceTests/ServiceDiscovery/KubernetesServiceDiscoveryTests.cs +++ b/test/Ocelot.AcceptanceTests/ServiceDiscovery/KubernetesServiceDiscoveryTests.cs @@ -1,5 +1,6 @@ using KubeClient; using KubeClient.Models; +using KubeClient.ResourceClients; using Microsoft.AspNetCore.Http; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; @@ -324,7 +325,7 @@ private void GivenThereIsAFakeKubernetesProvider(EndpointsV1 endpoints, bool isS } endpoints.Metadata.Generation = _k8sServiceGeneration; - json = JsonConvert.SerializeObject(endpoints); + json = JsonConvert.SerializeObject(endpoints, KubeResourceClient.SerializerSettings); } if (context.Request.Headers.TryGetValue("Authorization", out var values)) diff --git a/test/Ocelot.IntegrationTests/Ocelot.IntegrationTests.csproj b/test/Ocelot.IntegrationTests/Ocelot.IntegrationTests.csproj index 344f8c99f..137efff9a 100644 --- a/test/Ocelot.IntegrationTests/Ocelot.IntegrationTests.csproj +++ b/test/Ocelot.IntegrationTests/Ocelot.IntegrationTests.csproj @@ -54,14 +54,14 @@ - - - - - - - - - + + + + + + + + + diff --git a/test/Ocelot.UnitTests/Kubernetes/KubeTests.cs b/test/Ocelot.UnitTests/Kubernetes/KubeTests.cs index e30c44bc8..57e9b2f0d 100644 --- a/test/Ocelot.UnitTests/Kubernetes/KubeTests.cs +++ b/test/Ocelot.UnitTests/Kubernetes/KubeTests.cs @@ -7,7 +7,6 @@ using Ocelot.Logging; using Ocelot.Provider.Kubernetes; using Ocelot.Provider.Kubernetes.Interfaces; -using Ocelot.Testing; using Ocelot.Values; using System.Runtime.CompilerServices; @@ -19,6 +18,8 @@ namespace Ocelot.UnitTests.Kubernetes; /// public class KubeTests : FileUnitTest { + static JsonSerializerSettings JsonSerializerSettings => KubeClient.ResourceClients.KubeResourceClient.SerializerSettings; + private readonly Mock _factory; private readonly Mock _logger; @@ -43,6 +44,7 @@ public async Task Should_return_service_from_k8s() given.ClientOptions.ApiEndPoint.ToString(), given.ProviderOptions.KubeNamespace, given.ProviderOptions.KeyOfServiceInK8s, + responseStatusCode: HttpStatusCode.OK, endpoints, out Lazy receivedToken); @@ -54,6 +56,66 @@ public async Task Should_return_service_from_k8s() receivedToken.Value.ShouldBe($"Bearer {nameof(Should_return_service_from_k8s)}"); } + [Theory] + [InlineData(HttpStatusCode.BadRequest)] + [InlineData(HttpStatusCode.Forbidden)] + [InlineData(HttpStatusCode.InternalServerError)] + [InlineData(HttpStatusCode.NotFound)] + [Trait("PR", "2266")] + public async Task Should_not_return_service_from_k8s_when_k8s_api_returns_error_response(HttpStatusCode expectedStatusCode) + { + // Arrange + var given = GivenClientAndProvider(out var serviceBuilder); + serviceBuilder.Setup(x => x.BuildServices(It.IsAny(), It.IsAny())) + .Returns(new Service[] { new(nameof(Should_not_return_service_from_k8s_when_k8s_api_returns_error_response), new("localhost", 80), string.Empty, string.Empty, Array.Empty()) }); + + var endpoints = GivenEndpoints(); + using var kubernetes = GivenThereIsAFakeKubeServiceDiscoveryProvider( + given.ClientOptions.ApiEndPoint.ToString(), + given.ProviderOptions.KubeNamespace, + given.ProviderOptions.KeyOfServiceInK8s, + expectedStatusCode, + endpoints, + out Lazy receivedToken); + + string expectedKubeApiErrorMessage = GetKubeApiErrorMessage(serviceName: given.ProviderOptions.KeyOfServiceInK8s, given.ProviderOptions.KubeNamespace, expectedStatusCode); + string expectedLogMessage = $"Failed to retrieve v1/Endpoints '{given.ProviderOptions.KeyOfServiceInK8s}' in namespace '{given.ProviderOptions.KubeNamespace}': (HTTP.{expectedStatusCode}/Failure/{expectedStatusCode}): {expectedKubeApiErrorMessage}"; + _logger.Setup(logger => logger.LogError(It.IsAny>(), It.IsAny())) + .Callback((Func messageFactory, Exception exception) => + { + messageFactory.ShouldNotBeNull(); + + string logMessage = messageFactory(); + logMessage.ShouldNotBeNullOrWhiteSpace(); + + // This is a little fragile, as it may change if other entries are logged due to implementation changes. + // Unfortunately, the use of a factory delegate for the log message, combined with reuse of Kube's logger for Retry.OperationAsync makes this tricky to test any other way so this is probably the best we can do for now. + if (logMessage.StartsWith("Ocelot Retry strategy")) + { + return; + } + + logMessage.ShouldBe(expectedLogMessage); + + exception.ShouldNotBeNull(); + KubeApiException kubeApiException = exception.ShouldBeOfType(); + StatusV1 errorResponse = kubeApiException.Status; + errorResponse.Status.ShouldBe(StatusV1.FailureStatus); + errorResponse.Code.ShouldBe((int)expectedStatusCode); + errorResponse.Reason.ShouldBe(expectedStatusCode.ToString()); + errorResponse.Message.ShouldNotBeNullOrWhiteSpace(); + }) + .Verifiable($"IOcelotLogger.LogError() was not called."); + + // Act + var services = await given.Provider.GetAsync(); + + // Assert + services.ShouldNotBeNull().Count.ShouldBe(0); + receivedToken.Value.ShouldBe($"Bearer {nameof(Should_not_return_service_from_k8s_when_k8s_api_returns_error_response)}"); + _logger.Verify(); + } + [Fact] // This is not unit test! LoL :) This should be an integration test or even an acceptance test... [Trait("Bug", "2110")] public async Task Should_return_single_service_from_k8s_during_concurrent_calls() @@ -147,6 +209,12 @@ protected EndpointsV1 GivenEndpoints( protected IWebHost GivenThereIsAFakeKubeServiceDiscoveryProvider(string url, string namespaces, string serviceName, EndpointsV1 endpointEntries, out Lazy receivedToken) + { + return GivenThereIsAFakeKubeServiceDiscoveryProvider(url, namespaces, serviceName, HttpStatusCode.OK, endpointEntries, out receivedToken); + } + + protected IWebHost GivenThereIsAFakeKubeServiceDiscoveryProvider(string url, string namespaces, string serviceName, + HttpStatusCode responseStatusCode, EndpointsV1 endpointEntries, out Lazy receivedToken) { var token = string.Empty; receivedToken = new(() => token); @@ -155,14 +223,33 @@ Task ProcessKubernetesRequest(HttpContext context) { if (context.Request.Path.Value == $"/api/v1/namespaces/{namespaces}/endpoints/{serviceName}") { + string responseBody; + if (context.Request.Headers.TryGetValue("Authorization", out var values)) { token = values.First(); } - var json = JsonConvert.SerializeObject(endpointEntries); + if (responseStatusCode == HttpStatusCode.OK) + { + responseBody = JsonConvert.SerializeObject(endpointEntries, JsonSerializerSettings); + } + else + { + responseBody = JsonConvert.SerializeObject(new StatusV1 + { + Message = GetKubeApiErrorMessage(serviceName, namespaces, responseStatusCode), + Reason = responseStatusCode.ToString(), + Code = (int)responseStatusCode, + Status = StatusV1.FailureStatus, + + }, JsonSerializerSettings); + } + + context.Response.StatusCode = (int)responseStatusCode; context.Response.Headers.Append("Content-Type", "application/json"); - return context.Response.WriteAsync(json); + + return context.Response.WriteAsync(responseBody); } return Task.CompletedTask; @@ -179,4 +266,9 @@ Task ProcessKubernetesRequest(HttpContext context) host.Start(); return host; } + + private static string GetKubeApiErrorMessage(string serviceName, string kubeNamespace, HttpStatusCode responseStatusCode) + { + return $"Failed to retrieve v1/Endpoints '{serviceName}' in namespace '{kubeNamespace}' (HTTP.{responseStatusCode}/Failure/{responseStatusCode}): This is an error response for HTTP status code {(int)responseStatusCode} ('{responseStatusCode}') from the fake Kubernetes API."; + } } diff --git a/test/Ocelot.UnitTests/Ocelot.UnitTests.csproj b/test/Ocelot.UnitTests/Ocelot.UnitTests.csproj index 2ea46d83c..de8f3df09 100644 --- a/test/Ocelot.UnitTests/Ocelot.UnitTests.csproj +++ b/test/Ocelot.UnitTests/Ocelot.UnitTests.csproj @@ -77,14 +77,14 @@ - - - - - - - - + + + + + + + + @@ -92,6 +92,6 @@ - +