From ad2f5ec5f5c603f2090ac1aa2e282e84343b8909 Mon Sep 17 00:00:00 2001 From: Adam Friedman Date: Sat, 15 Mar 2025 14:01:39 +1100 Subject: [PATCH 01/11] Upgrade to KubeClient v3 and log failed Kubernetes API requests KubeClient v3 should simplify dependency-management, from Ocelot's perspective: * KubeClient no longer has an external dependency (HTTPlease) for its HttpClient support. The internalised version of this support is now located in the KubeClient.Http namespace. * KubeClient v3 is multi-targeted for net9.0, as well as a couple of older versions that some consumers are still using (support for these older versions will be tailed off over the next couple of minor releases) Related discussion at: tintoy/dotnet-kube-client#166 tintoy/dotnet-kube-client#167 --- .../EndPointClientV1.cs | 37 +++++++++++++++---- .../Ocelot.Provider.Kubernetes.csproj | 4 +- 2 files changed, 32 insertions(+), 9 deletions(-) diff --git a/src/Ocelot.Provider.Kubernetes/EndPointClientV1.cs b/src/Ocelot.Provider.Kubernetes/EndPointClientV1.cs index 6b81693d4..adc5add73 100644 --- a/src/Ocelot.Provider.Kubernetes/EndPointClientV1.cs +++ b/src/Ocelot.Provider.Kubernetes/EndPointClientV1.cs @@ -1,17 +1,20 @@ -using HTTPlease; +using KubeClient.Http; using KubeClient.Models; using KubeClient.ResourceClients; +using Microsoft.Extensions.Logging; using Ocelot.Provider.Kubernetes.Interfaces; namespace Ocelot.Provider.Kubernetes; public class EndPointClientV1 : KubeResourceClient, IEndPointClient { - private readonly HttpRequest _collection; + private static readonly HttpRequest Collection = KubeRequest.Create("api/v1/namespaces/{Namespace}/endpoints/{ServiceName}"); + + private readonly ILogger _logger; public EndPointClientV1(IKubeApiClient client) : base(client) { - _collection = KubeRequest.Create("api/v1/namespaces/{Namespace}/endpoints/{ServiceName}"); + _logger = client.LoggerFactory.CreateLogger(); } public async Task GetAsync(string serviceName, string kubeNamespace = null, CancellationToken cancellationToken = default) @@ -21,7 +24,7 @@ public async Task GetAsync(string serviceName, string kubeNamespace throw new ArgumentNullException(nameof(serviceName)); } - var request = _collection + var request = Collection .WithTemplateParameters(new { Namespace = kubeNamespace ?? KubeClient.DefaultNamespace, @@ -30,8 +33,28 @@ public async Task GetAsync(string serviceName, string kubeNamespace var response = await Http.GetAsync(request, cancellationToken); - return response.IsSuccessStatusCode - ? await response.ReadContentAsAsync() - : null; + if (!response.IsSuccessStatusCode) + { + StatusV1 errorResponse = await response.ReadContentAsAsync(); + + if (_logger.IsEnabled(LogLevel.Debug)) + { + (string resourceKind, string resourceApiVersion) = KubeObjectV1.GetKubeKind(); + _logger.LogDebug("Failed to retrieve {ResourceApiVersion}/{ResourceKind} {ResourceName} in namespace {ResourceNamespace} ({HttpStatusCode}/{Status}/{StatusReason}): {StatusMessage}", + resourceApiVersion, + resourceKind, + serviceName, + kubeNamespace, + response.StatusCode, + errorResponse.Status, + errorResponse.Reason, + errorResponse.Message + ); + } + + return null; + } + + return await response.ReadContentAsAsync(); } } diff --git a/src/Ocelot.Provider.Kubernetes/Ocelot.Provider.Kubernetes.csproj b/src/Ocelot.Provider.Kubernetes/Ocelot.Provider.Kubernetes.csproj index 711ff0e80..d15489b04 100644 --- a/src/Ocelot.Provider.Kubernetes/Ocelot.Provider.Kubernetes.csproj +++ b/src/Ocelot.Provider.Kubernetes/Ocelot.Provider.Kubernetes.csproj @@ -25,8 +25,8 @@ 1591 - - + + all From fd98410656f98c2e0731e03aa768f5dadf0ea5f0 Mon Sep 17 00:00:00 2001 From: Adam Friedman Date: Sat, 15 Mar 2025 14:09:16 +1100 Subject: [PATCH 02/11] Don't attempt to deserialise response payload from Kubernetes API if it isn't going to be used --- src/Ocelot.Provider.Kubernetes/EndPointClientV1.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Ocelot.Provider.Kubernetes/EndPointClientV1.cs b/src/Ocelot.Provider.Kubernetes/EndPointClientV1.cs index adc5add73..66c390c9d 100644 --- a/src/Ocelot.Provider.Kubernetes/EndPointClientV1.cs +++ b/src/Ocelot.Provider.Kubernetes/EndPointClientV1.cs @@ -35,10 +35,10 @@ public async Task GetAsync(string serviceName, string kubeNamespace if (!response.IsSuccessStatusCode) { - StatusV1 errorResponse = await response.ReadContentAsAsync(); - if (_logger.IsEnabled(LogLevel.Debug)) { + StatusV1 errorResponse = await response.ReadContentAsAsync(); + (string resourceKind, string resourceApiVersion) = KubeObjectV1.GetKubeKind(); _logger.LogDebug("Failed to retrieve {ResourceApiVersion}/{ResourceKind} {ResourceName} in namespace {ResourceNamespace} ({HttpStatusCode}/{Status}/{StatusReason}): {StatusMessage}", resourceApiVersion, From 4b5a6ef3f0fc9a6634c5ee85193db0fb2f1d3d56 Mon Sep 17 00:00:00 2001 From: Adam Friedman Date: Sun, 16 Mar 2025 11:31:20 +1100 Subject: [PATCH 03/11] Simplify logging implementation by moving it out of EndPointClientV1 and into its caller --- .../EndPointClientV1.cs | 31 +------------- src/Ocelot.Provider.Kubernetes/Kube.cs | 41 +++++++++++++++++-- 2 files changed, 40 insertions(+), 32 deletions(-) diff --git a/src/Ocelot.Provider.Kubernetes/EndPointClientV1.cs b/src/Ocelot.Provider.Kubernetes/EndPointClientV1.cs index 66c390c9d..e94c1a3d1 100644 --- a/src/Ocelot.Provider.Kubernetes/EndPointClientV1.cs +++ b/src/Ocelot.Provider.Kubernetes/EndPointClientV1.cs @@ -1,7 +1,6 @@ using KubeClient.Http; using KubeClient.Models; using KubeClient.ResourceClients; -using Microsoft.Extensions.Logging; using Ocelot.Provider.Kubernetes.Interfaces; namespace Ocelot.Provider.Kubernetes; @@ -10,11 +9,8 @@ public class EndPointClientV1 : KubeResourceClient, IEndPointClient { private static readonly HttpRequest Collection = KubeRequest.Create("api/v1/namespaces/{Namespace}/endpoints/{ServiceName}"); - private readonly ILogger _logger; - public EndPointClientV1(IKubeApiClient client) : base(client) { - _logger = client.LoggerFactory.CreateLogger(); } public async Task GetAsync(string serviceName, string kubeNamespace = null, CancellationToken cancellationToken = default) @@ -31,30 +27,7 @@ public async Task GetAsync(string serviceName, string kubeNamespace ServiceName = serviceName, }); - var response = await Http.GetAsync(request, cancellationToken); - - if (!response.IsSuccessStatusCode) - { - if (_logger.IsEnabled(LogLevel.Debug)) - { - StatusV1 errorResponse = await response.ReadContentAsAsync(); - - (string resourceKind, string resourceApiVersion) = KubeObjectV1.GetKubeKind(); - _logger.LogDebug("Failed to retrieve {ResourceApiVersion}/{ResourceKind} {ResourceName} in namespace {ResourceNamespace} ({HttpStatusCode}/{Status}/{StatusReason}): {StatusMessage}", - resourceApiVersion, - resourceKind, - serviceName, - kubeNamespace, - response.StatusCode, - errorResponse.Status, - errorResponse.Reason, - errorResponse.Message - ); - } - - return null; - } - - return await response.ReadContentAsAsync(); + return await Http.GetAsync(request, cancellationToken) + .ReadContentAsObjectV1Async(operationDescription: $"get {nameof(EndpointsV1)}"); } } diff --git a/src/Ocelot.Provider.Kubernetes/Kube.cs b/src/Ocelot.Provider.Kubernetes/Kube.cs index 6ae36bb71..5f62eaf56 100644 --- a/src/Ocelot.Provider.Kubernetes/Kube.cs +++ b/src/Ocelot.Provider.Kubernetes/Kube.cs @@ -15,6 +15,8 @@ namespace Ocelot.Provider.Kubernetes; /// public class Kube : IServiceDiscoveryProvider { + private static readonly (string ResourceKind, string ResourceApiVersion) EndPointsKubeKind = KubeObjectV1.GetKubeKind(); + private readonly KubeRegistryConfiguration _configuration; private readonly IOcelotLogger _logger; private readonly IKubeApiClient _kubeApi; @@ -46,9 +48,42 @@ public virtual async Task> GetAsync() .ToList(); } - private Task GetEndpoint() => _kubeApi - .ResourceClient(client => new EndPointClientV1(client)) - .GetAsync(_configuration.KeyOfServiceInK8s, _configuration.KubeNamespace); + private async Task GetEndpoint() + { + string serviceName = _configuration.KeyOfServiceInK8s; + string kubeNamespace = _configuration.KubeNamespace; + + try + { + return await _kubeApi + .ResourceClient(client => new EndPointClientV1(client)) + .GetAsync(serviceName, kubeNamespace); + } + catch (KubeApiException kubeApiError) + { + _logger.LogError(() => + { + StatusV1 errorResponse = kubeApiError.Status; + string httpStatusCode = "Unknown"; + if (kubeApiError.InnerException is HttpRequestException httpRequestError) + { + httpStatusCode = httpRequestError.StatusCode.ToString(); + } + + return $"Failed to retrieve {EndPointsKubeKind.ResourceApiVersion}/{EndPointsKubeKind.ResourceKind} {serviceName} in namespace {kubeNamespace} ({httpStatusCode}/{errorResponse.Status}/{errorResponse.Reason}): {errorResponse.Message}"; + }, kubeApiError); + } + catch (HttpRequestException unexpectedRequestError) + { + _logger.LogError(() => $"Failed to retrieve {EndPointsKubeKind.ResourceApiVersion}/{EndPointsKubeKind.ResourceKind} {serviceName} in namespace {kubeNamespace} ({unexpectedRequestError.HttpRequestError}/{unexpectedRequestError.StatusCode}).", unexpectedRequestError); + } + catch (Exception unexpectedError) + { + _logger.LogError(() => $"Failed to retrieve {EndPointsKubeKind.ResourceApiVersion}/{EndPointsKubeKind.ResourceKind} {serviceName} in namespace {kubeNamespace} (an unexpected error occurred).", unexpectedError); + } + + return null; + } private bool CheckErroneousState(EndpointsV1 endpoint) => (endpoint?.Subsets?.Count ?? 0) == 0; // null or count is zero From 9a8bf9cd9708a20c8d6296d7a7ea90754928c5f2 Mon Sep 17 00:00:00 2001 From: Adam Friedman Date: Sun, 16 Mar 2025 12:32:32 +1100 Subject: [PATCH 04/11] Add support for returning error responses from the fake Kubernetes API used by service-discovery tests --- .../KubernetesServiceDiscoveryTests.cs | 49 +++++++++++++------ 1 file changed, 35 insertions(+), 14 deletions(-) diff --git a/test/Ocelot.AcceptanceTests/ServiceDiscovery/KubernetesServiceDiscoveryTests.cs b/test/Ocelot.AcceptanceTests/ServiceDiscovery/KubernetesServiceDiscoveryTests.cs index 9b6a5c6da..ea4e9dc99 100644 --- a/test/Ocelot.AcceptanceTests/ServiceDiscovery/KubernetesServiceDiscoveryTests.cs +++ b/test/Ocelot.AcceptanceTests/ServiceDiscovery/KubernetesServiceDiscoveryTests.cs @@ -1,5 +1,6 @@ using KubeClient; using KubeClient.Models; +using KubeClient.ResourceClients; using Microsoft.AspNetCore.Http; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; @@ -147,7 +148,7 @@ public void ShouldHighlyLoadOnUnstableKubeProvider_WithRoundRobinLoadBalancing(i { int failPerThreads = (totalRequests / k8sGeneration) - 1; // k8sGeneration means number of offline services var (endpoints, servicePorts) = ArrangeHighLoadOnKubeProviderAndRoundRobinBalancer(totalServices); - GivenThereIsAFakeKubernetesProvider(endpoints, false, k8sGeneration, failPerThreads); // false means unstable, k8sGeneration services will be removed from the list + GivenThereIsAFakeKubernetesProvider(endpoints, HttpStatusCode.OK, false, k8sGeneration, failPerThreads); // false means unstable, k8sGeneration services will be removed from the list HighlyLoadOnKubeProviderAndRoundRobinBalancer(totalRequests, k8sGeneration); @@ -293,9 +294,13 @@ private FileConfiguration GivenKubeConfiguration(string serviceNamespace, FileRo private void GivenThereIsAFakeKubernetesProvider(EndpointsV1 endpoints, [CallerMemberName] string serviceName = nameof(KubernetesServiceDiscoveryTests), string namespaces = nameof(KubernetesServiceDiscoveryTests)) - => GivenThereIsAFakeKubernetesProvider(endpoints, true, 0, 0, serviceName, namespaces); + => GivenThereIsAFakeKubernetesProvider(endpoints, responseStatusCode: HttpStatusCode.OK, serviceName, namespaces); - private void GivenThereIsAFakeKubernetesProvider(EndpointsV1 endpoints, bool isStable, int offlineServicesNo, int offlinePerThreads, + private void GivenThereIsAFakeKubernetesProvider(EndpointsV1 endpoints, HttpStatusCode responseStatusCode, + [CallerMemberName] string serviceName = nameof(KubernetesServiceDiscoveryTests), string namespaces = nameof(KubernetesServiceDiscoveryTests)) + => GivenThereIsAFakeKubernetesProvider(endpoints, responseStatusCode, isStable: true, offlineServicesNo: 0, offlinePerThreads: 0, serviceName, namespaces); + + private void GivenThereIsAFakeKubernetesProvider(EndpointsV1 endpoints, HttpStatusCode responseStatusCode, bool isStable, int offlineServicesNo, int offlinePerThreads, [CallerMemberName] string serviceName = nameof(KubernetesServiceDiscoveryTests), string namespaces = nameof(KubernetesServiceDiscoveryTests)) { _k8sCounter = 0; @@ -308,23 +313,38 @@ private void GivenThereIsAFakeKubernetesProvider(EndpointsV1 endpoints, bool isS lock (K8sCounterLocker) { _k8sCounter++; - var subset = endpoints.Subsets[0]; - // Each offlinePerThreads-th request to integrated K8s endpoint should fail - if (!isStable && _k8sCounter % offlinePerThreads == 0 && _k8sCounter >= offlinePerThreads) + if (responseStatusCode == HttpStatusCode.OK) { - while (offlineServicesNo-- > 0) + var subset = endpoints.Subsets[0]; + + // Each offlinePerThreads-th request to integrated K8s endpoint should fail + if (!isStable && _k8sCounter % offlinePerThreads == 0 && _k8sCounter >= offlinePerThreads) { - int index = subset.Addresses.Count - 1; // Random.Shared.Next(0, subset.Addresses.Count - 1); - subset.Addresses.RemoveAt(index); - subset.Ports.RemoveAt(index); + while (offlineServicesNo-- > 0) + { + int index = subset.Addresses.Count - 1; // Random.Shared.Next(0, subset.Addresses.Count - 1); + subset.Addresses.RemoveAt(index); + subset.Ports.RemoveAt(index); + } + + _k8sServiceGeneration++; } - _k8sServiceGeneration++; + endpoints.Metadata.Generation = _k8sServiceGeneration; + json = JsonConvert.SerializeObject(endpoints, KubeResourceClient.SerializerSettings); + } + else + { + StatusV1 errorResponse = new() + { + Code = context.Response.StatusCode, + Reason = responseStatusCode.ToString(), + Message = "This is an error response from the fake Kubernetes API.", + Status = StatusV1.FailureStatus, + }; + json = JsonConvert.SerializeObject(_k8sServiceGeneration, KubeResourceClient.SerializerSettings); } - - endpoints.Metadata.Generation = _k8sServiceGeneration; - json = JsonConvert.SerializeObject(endpoints); } if (context.Request.Headers.TryGetValue("Authorization", out var values)) @@ -332,6 +352,7 @@ private void GivenThereIsAFakeKubernetesProvider(EndpointsV1 endpoints, bool isS _receivedToken = values.First(); } + context.Response.StatusCode = (int)responseStatusCode; context.Response.Headers.Append("Content-Type", "application/json"); await context.Response.WriteAsync(json); } From 8472f4a26e3cda879ce1bb424de9351cb2718537 Mon Sep 17 00:00:00 2001 From: Adam Friedman Date: Mon, 17 Mar 2025 09:39:33 +1100 Subject: [PATCH 05/11] Revert "Add support for returning error responses from the fake Kubernetes API used by service-discovery tests" This reverts commit 9a8bf9cd9708a20c8d6296d7a7ea90754928c5f2. --- .../KubernetesServiceDiscoveryTests.cs | 49 ++++++------------- 1 file changed, 14 insertions(+), 35 deletions(-) diff --git a/test/Ocelot.AcceptanceTests/ServiceDiscovery/KubernetesServiceDiscoveryTests.cs b/test/Ocelot.AcceptanceTests/ServiceDiscovery/KubernetesServiceDiscoveryTests.cs index ea4e9dc99..9b6a5c6da 100644 --- a/test/Ocelot.AcceptanceTests/ServiceDiscovery/KubernetesServiceDiscoveryTests.cs +++ b/test/Ocelot.AcceptanceTests/ServiceDiscovery/KubernetesServiceDiscoveryTests.cs @@ -1,6 +1,5 @@ using KubeClient; using KubeClient.Models; -using KubeClient.ResourceClients; using Microsoft.AspNetCore.Http; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; @@ -148,7 +147,7 @@ public void ShouldHighlyLoadOnUnstableKubeProvider_WithRoundRobinLoadBalancing(i { int failPerThreads = (totalRequests / k8sGeneration) - 1; // k8sGeneration means number of offline services var (endpoints, servicePorts) = ArrangeHighLoadOnKubeProviderAndRoundRobinBalancer(totalServices); - GivenThereIsAFakeKubernetesProvider(endpoints, HttpStatusCode.OK, false, k8sGeneration, failPerThreads); // false means unstable, k8sGeneration services will be removed from the list + GivenThereIsAFakeKubernetesProvider(endpoints, false, k8sGeneration, failPerThreads); // false means unstable, k8sGeneration services will be removed from the list HighlyLoadOnKubeProviderAndRoundRobinBalancer(totalRequests, k8sGeneration); @@ -294,13 +293,9 @@ private FileConfiguration GivenKubeConfiguration(string serviceNamespace, FileRo private void GivenThereIsAFakeKubernetesProvider(EndpointsV1 endpoints, [CallerMemberName] string serviceName = nameof(KubernetesServiceDiscoveryTests), string namespaces = nameof(KubernetesServiceDiscoveryTests)) - => GivenThereIsAFakeKubernetesProvider(endpoints, responseStatusCode: HttpStatusCode.OK, serviceName, namespaces); + => GivenThereIsAFakeKubernetesProvider(endpoints, true, 0, 0, serviceName, namespaces); - private void GivenThereIsAFakeKubernetesProvider(EndpointsV1 endpoints, HttpStatusCode responseStatusCode, - [CallerMemberName] string serviceName = nameof(KubernetesServiceDiscoveryTests), string namespaces = nameof(KubernetesServiceDiscoveryTests)) - => GivenThereIsAFakeKubernetesProvider(endpoints, responseStatusCode, isStable: true, offlineServicesNo: 0, offlinePerThreads: 0, serviceName, namespaces); - - private void GivenThereIsAFakeKubernetesProvider(EndpointsV1 endpoints, HttpStatusCode responseStatusCode, bool isStable, int offlineServicesNo, int offlinePerThreads, + private void GivenThereIsAFakeKubernetesProvider(EndpointsV1 endpoints, bool isStable, int offlineServicesNo, int offlinePerThreads, [CallerMemberName] string serviceName = nameof(KubernetesServiceDiscoveryTests), string namespaces = nameof(KubernetesServiceDiscoveryTests)) { _k8sCounter = 0; @@ -313,38 +308,23 @@ private void GivenThereIsAFakeKubernetesProvider(EndpointsV1 endpoints, HttpStat lock (K8sCounterLocker) { _k8sCounter++; + var subset = endpoints.Subsets[0]; - if (responseStatusCode == HttpStatusCode.OK) + // Each offlinePerThreads-th request to integrated K8s endpoint should fail + if (!isStable && _k8sCounter % offlinePerThreads == 0 && _k8sCounter >= offlinePerThreads) { - var subset = endpoints.Subsets[0]; - - // Each offlinePerThreads-th request to integrated K8s endpoint should fail - if (!isStable && _k8sCounter % offlinePerThreads == 0 && _k8sCounter >= offlinePerThreads) + while (offlineServicesNo-- > 0) { - while (offlineServicesNo-- > 0) - { - int index = subset.Addresses.Count - 1; // Random.Shared.Next(0, subset.Addresses.Count - 1); - subset.Addresses.RemoveAt(index); - subset.Ports.RemoveAt(index); - } - - _k8sServiceGeneration++; + int index = subset.Addresses.Count - 1; // Random.Shared.Next(0, subset.Addresses.Count - 1); + subset.Addresses.RemoveAt(index); + subset.Ports.RemoveAt(index); } - endpoints.Metadata.Generation = _k8sServiceGeneration; - json = JsonConvert.SerializeObject(endpoints, KubeResourceClient.SerializerSettings); - } - else - { - StatusV1 errorResponse = new() - { - Code = context.Response.StatusCode, - Reason = responseStatusCode.ToString(), - Message = "This is an error response from the fake Kubernetes API.", - Status = StatusV1.FailureStatus, - }; - json = JsonConvert.SerializeObject(_k8sServiceGeneration, KubeResourceClient.SerializerSettings); + _k8sServiceGeneration++; } + + endpoints.Metadata.Generation = _k8sServiceGeneration; + json = JsonConvert.SerializeObject(endpoints); } if (context.Request.Headers.TryGetValue("Authorization", out var values)) @@ -352,7 +332,6 @@ private void GivenThereIsAFakeKubernetesProvider(EndpointsV1 endpoints, HttpStat _receivedToken = values.First(); } - context.Response.StatusCode = (int)responseStatusCode; context.Response.Headers.Append("Content-Type", "application/json"); await context.Response.WriteAsync(json); } From 8a4faa347167d09dcc12a04e25a4bb6284bd674a Mon Sep 17 00:00:00 2001 From: Adam Friedman Date: Sun, 16 Mar 2025 12:23:37 +1100 Subject: [PATCH 06/11] Upgrade KubeClient to v3.0.1 --- .../Ocelot.Provider.Kubernetes.csproj | 4 ++-- .../Ocelot.AcceptanceTests.csproj | 18 +++++++++--------- .../Ocelot.IntegrationTests.csproj | 18 +++++++++--------- test/Ocelot.UnitTests/Ocelot.UnitTests.csproj | 18 +++++++++--------- 4 files changed, 29 insertions(+), 29 deletions(-) diff --git a/src/Ocelot.Provider.Kubernetes/Ocelot.Provider.Kubernetes.csproj b/src/Ocelot.Provider.Kubernetes/Ocelot.Provider.Kubernetes.csproj index d15489b04..d2ad0acf7 100644 --- a/src/Ocelot.Provider.Kubernetes/Ocelot.Provider.Kubernetes.csproj +++ b/src/Ocelot.Provider.Kubernetes/Ocelot.Provider.Kubernetes.csproj @@ -25,8 +25,8 @@ 1591 - - + + all diff --git a/test/Ocelot.AcceptanceTests/Ocelot.AcceptanceTests.csproj b/test/Ocelot.AcceptanceTests/Ocelot.AcceptanceTests.csproj index 2e88aaaee..ec8d0e8b1 100644 --- a/test/Ocelot.AcceptanceTests/Ocelot.AcceptanceTests.csproj +++ b/test/Ocelot.AcceptanceTests/Ocelot.AcceptanceTests.csproj @@ -73,14 +73,14 @@ all - - - - - - - - + + + + + + + + @@ -88,6 +88,6 @@ - + diff --git a/test/Ocelot.IntegrationTests/Ocelot.IntegrationTests.csproj b/test/Ocelot.IntegrationTests/Ocelot.IntegrationTests.csproj index 344f8c99f..137efff9a 100644 --- a/test/Ocelot.IntegrationTests/Ocelot.IntegrationTests.csproj +++ b/test/Ocelot.IntegrationTests/Ocelot.IntegrationTests.csproj @@ -54,14 +54,14 @@ - - - - - - - - - + + + + + + + + + diff --git a/test/Ocelot.UnitTests/Ocelot.UnitTests.csproj b/test/Ocelot.UnitTests/Ocelot.UnitTests.csproj index 2ea46d83c..de8f3df09 100644 --- a/test/Ocelot.UnitTests/Ocelot.UnitTests.csproj +++ b/test/Ocelot.UnitTests/Ocelot.UnitTests.csproj @@ -77,14 +77,14 @@ - - - - - - - - + + + + + + + + @@ -92,6 +92,6 @@ - + From 6c1ac8a4226be7b84a210546fed9412993187ca7 Mon Sep 17 00:00:00 2001 From: Adam Friedman Date: Mon, 17 Mar 2025 09:40:57 +1100 Subject: [PATCH 07/11] Use consistent serialisation settings for fake Kubernetes API responses --- .../ServiceDiscovery/KubernetesServiceDiscoveryTests.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/Ocelot.AcceptanceTests/ServiceDiscovery/KubernetesServiceDiscoveryTests.cs b/test/Ocelot.AcceptanceTests/ServiceDiscovery/KubernetesServiceDiscoveryTests.cs index 9b6a5c6da..fa3841ec8 100644 --- a/test/Ocelot.AcceptanceTests/ServiceDiscovery/KubernetesServiceDiscoveryTests.cs +++ b/test/Ocelot.AcceptanceTests/ServiceDiscovery/KubernetesServiceDiscoveryTests.cs @@ -1,5 +1,6 @@ using KubeClient; using KubeClient.Models; +using KubeClient.ResourceClients; using Microsoft.AspNetCore.Http; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; @@ -324,7 +325,7 @@ private void GivenThereIsAFakeKubernetesProvider(EndpointsV1 endpoints, bool isS } endpoints.Metadata.Generation = _k8sServiceGeneration; - json = JsonConvert.SerializeObject(endpoints); + json = JsonConvert.SerializeObject(endpoints, KubeResourceClient.SerializerSettings); } if (context.Request.Headers.TryGetValue("Authorization", out var values)) From 6a6c37405c9e8b4b076dd3ad63f753e0adccb5e2 Mon Sep 17 00:00:00 2001 From: Adam Friedman Date: Mon, 17 Mar 2025 10:58:08 +1100 Subject: [PATCH 08/11] Add unit tests for handling of error responses from Kubernetes API --- src/Ocelot.Provider.Kubernetes/Kube.cs | 6 +- test/Ocelot.UnitTests/Kubernetes/KubeTests.cs | 98 ++++++++++++++++++- 2 files changed, 98 insertions(+), 6 deletions(-) diff --git a/src/Ocelot.Provider.Kubernetes/Kube.cs b/src/Ocelot.Provider.Kubernetes/Kube.cs index 5f62eaf56..511ad93f8 100644 --- a/src/Ocelot.Provider.Kubernetes/Kube.cs +++ b/src/Ocelot.Provider.Kubernetes/Kube.cs @@ -70,16 +70,16 @@ private async Task GetEndpoint() httpStatusCode = httpRequestError.StatusCode.ToString(); } - return $"Failed to retrieve {EndPointsKubeKind.ResourceApiVersion}/{EndPointsKubeKind.ResourceKind} {serviceName} in namespace {kubeNamespace} ({httpStatusCode}/{errorResponse.Status}/{errorResponse.Reason}): {errorResponse.Message}"; + return $"Failed to retrieve {EndPointsKubeKind.ResourceApiVersion}/{EndPointsKubeKind.ResourceKind} '{serviceName}' in namespace '{kubeNamespace}' (HTTP.{httpStatusCode}/{errorResponse.Status}/{errorResponse.Reason}): {errorResponse.Message}"; }, kubeApiError); } catch (HttpRequestException unexpectedRequestError) { - _logger.LogError(() => $"Failed to retrieve {EndPointsKubeKind.ResourceApiVersion}/{EndPointsKubeKind.ResourceKind} {serviceName} in namespace {kubeNamespace} ({unexpectedRequestError.HttpRequestError}/{unexpectedRequestError.StatusCode}).", unexpectedRequestError); + _logger.LogError(() => $"Failed to retrieve {EndPointsKubeKind.ResourceApiVersion}/{EndPointsKubeKind.ResourceKind} '{serviceName}' in namespace '{kubeNamespace}' ({unexpectedRequestError.HttpRequestError}/HTTP.{unexpectedRequestError.StatusCode}).", unexpectedRequestError); } catch (Exception unexpectedError) { - _logger.LogError(() => $"Failed to retrieve {EndPointsKubeKind.ResourceApiVersion}/{EndPointsKubeKind.ResourceKind} {serviceName} in namespace {kubeNamespace} (an unexpected error occurred).", unexpectedError); + _logger.LogError(() => $"Failed to retrieve {EndPointsKubeKind.ResourceApiVersion}/{EndPointsKubeKind.ResourceKind} '{serviceName}' in namespace '{kubeNamespace}' (an unexpected error occurred).", unexpectedError); } return null; diff --git a/test/Ocelot.UnitTests/Kubernetes/KubeTests.cs b/test/Ocelot.UnitTests/Kubernetes/KubeTests.cs index e30c44bc8..0eae1eee3 100644 --- a/test/Ocelot.UnitTests/Kubernetes/KubeTests.cs +++ b/test/Ocelot.UnitTests/Kubernetes/KubeTests.cs @@ -7,7 +7,6 @@ using Ocelot.Logging; using Ocelot.Provider.Kubernetes; using Ocelot.Provider.Kubernetes.Interfaces; -using Ocelot.Testing; using Ocelot.Values; using System.Runtime.CompilerServices; @@ -19,6 +18,8 @@ namespace Ocelot.UnitTests.Kubernetes; /// public class KubeTests : FileUnitTest { + static JsonSerializerSettings JsonSerializerSettings => KubeClient.ResourceClients.KubeResourceClient.SerializerSettings; + private readonly Mock _factory; private readonly Mock _logger; @@ -43,6 +44,7 @@ public async Task Should_return_service_from_k8s() given.ClientOptions.ApiEndPoint.ToString(), given.ProviderOptions.KubeNamespace, given.ProviderOptions.KeyOfServiceInK8s, + responseStatusCode: HttpStatusCode.OK, endpoints, out Lazy receivedToken); @@ -54,6 +56,66 @@ public async Task Should_return_service_from_k8s() receivedToken.Value.ShouldBe($"Bearer {nameof(Should_return_service_from_k8s)}"); } + [Theory] + [InlineData(HttpStatusCode.BadRequest)] + [InlineData(HttpStatusCode.Forbidden)] + [InlineData(HttpStatusCode.InternalServerError)] + [InlineData(HttpStatusCode.NotFound)] + [Trait("Feat", "345")] + public async Task Should_not_return_service_from_k8s_when_k8s_api_returns_error_response(HttpStatusCode expectedStatusCode) + { + // Arrange + var given = GivenClientAndProvider(out var serviceBuilder); + serviceBuilder.Setup(x => x.BuildServices(It.IsAny(), It.IsAny())) + .Returns(new Service[] { new(nameof(Should_return_service_from_k8s), new("localhost", 80), string.Empty, string.Empty, Array.Empty()) }); + + var endpoints = GivenEndpoints(); + using var kubernetes = GivenThereIsAFakeKubeServiceDiscoveryProvider( + given.ClientOptions.ApiEndPoint.ToString(), + given.ProviderOptions.KubeNamespace, + given.ProviderOptions.KeyOfServiceInK8s, + expectedStatusCode, + endpoints, + out Lazy receivedToken); + + string expectedKubeApiErrorMessage = GetKubeApiErrorMessage(serviceName: given.ProviderOptions.KeyOfServiceInK8s, given.ProviderOptions.KubeNamespace, expectedStatusCode); + string expectedLogMessage = $"Failed to retrieve v1/Endpoints '{given.ProviderOptions.KeyOfServiceInK8s}' in namespace '{given.ProviderOptions.KubeNamespace}' (HTTP.{expectedStatusCode}/Failure/{expectedStatusCode}): {expectedKubeApiErrorMessage}"; + _logger.Setup(logger => logger.LogError(It.IsAny>(), It.IsAny())) + .Callback((Func messageFactory, Exception exception) => + { + messageFactory.ShouldNotBeNull(); + + string logMessage = messageFactory(); + logMessage.ShouldNotBeNullOrWhiteSpace(); + + // This is a little fragile, as it may change if other entries are logged due to implementation changes. + // Unfortunately, the use of a factory delegate for the log message, combined with reuse of Kube's logger for Retry.OperationAsync makes this tricky to test any other way so this is probably the best we can do for now. + if (logMessage.StartsWith("Ocelot Retry strategy")) + { + return; + } + + logMessage.ShouldBe(expectedLogMessage); + + exception.ShouldNotBeNull(); + KubeApiException kubeApiException = exception.ShouldBeOfType(); + StatusV1 errorResponse = kubeApiException.Status; + errorResponse.Status.ShouldBe(StatusV1.FailureStatus); + errorResponse.Code.ShouldBe((int)expectedStatusCode); + errorResponse.Reason.ShouldBe(expectedStatusCode.ToString()); + errorResponse.Message.ShouldNotBeNullOrWhiteSpace(); + }) + .Verifiable($"IOcelotLogger.LogError() was not called."); + + // Act + var services = await given.Provider.GetAsync(); + + // Assert + services.ShouldNotBeNull().Count.ShouldBe(0); + receivedToken.Value.ShouldBe($"Bearer {nameof(Should_not_return_service_from_k8s_when_k8s_api_returns_error_response)}"); + _logger.Verify(); + } + [Fact] // This is not unit test! LoL :) This should be an integration test or even an acceptance test... [Trait("Bug", "2110")] public async Task Should_return_single_service_from_k8s_during_concurrent_calls() @@ -147,6 +209,12 @@ protected EndpointsV1 GivenEndpoints( protected IWebHost GivenThereIsAFakeKubeServiceDiscoveryProvider(string url, string namespaces, string serviceName, EndpointsV1 endpointEntries, out Lazy receivedToken) + { + return GivenThereIsAFakeKubeServiceDiscoveryProvider(url, namespaces, serviceName, HttpStatusCode.OK, endpointEntries, out receivedToken); + } + + protected IWebHost GivenThereIsAFakeKubeServiceDiscoveryProvider(string url, string namespaces, string serviceName, + HttpStatusCode responseStatusCode, EndpointsV1 endpointEntries, out Lazy receivedToken) { var token = string.Empty; receivedToken = new(() => token); @@ -155,14 +223,33 @@ Task ProcessKubernetesRequest(HttpContext context) { if (context.Request.Path.Value == $"/api/v1/namespaces/{namespaces}/endpoints/{serviceName}") { + string responseBody; + if (context.Request.Headers.TryGetValue("Authorization", out var values)) { token = values.First(); } - var json = JsonConvert.SerializeObject(endpointEntries); + if (responseStatusCode == HttpStatusCode.OK) + { + responseBody = JsonConvert.SerializeObject(endpointEntries, JsonSerializerSettings); + } + else + { + responseBody = JsonConvert.SerializeObject(new StatusV1 + { + Message = GetKubeApiErrorMessage(serviceName, namespaces, responseStatusCode), + Reason = responseStatusCode.ToString(), + Code = (int)responseStatusCode, + Status = StatusV1.FailureStatus, + + }, JsonSerializerSettings); + } + + context.Response.StatusCode = (int)responseStatusCode; context.Response.Headers.Append("Content-Type", "application/json"); - return context.Response.WriteAsync(json); + + return context.Response.WriteAsync(responseBody); } return Task.CompletedTask; @@ -179,4 +266,9 @@ Task ProcessKubernetesRequest(HttpContext context) host.Start(); return host; } + + private static string GetKubeApiErrorMessage(string serviceName, string kubeNamespace, HttpStatusCode responseStatusCode) + { + return $"Failed to retrieve v1/Endpoints '{serviceName}' in namespace '{kubeNamespace}' (HTTP.{responseStatusCode}/Failure/{responseStatusCode}): This is an error response for HTTP status code {(int)responseStatusCode} ('{responseStatusCode}') from the fake Kubernetes API."; + } } From c853a6f25ffaef8c2174b4e6ac6fdb8ce064ad2d Mon Sep 17 00:00:00 2001 From: Raman Maksimchuk Date: Mon, 17 Mar 2025 14:43:56 +0300 Subject: [PATCH 09/11] code review by @raman-m --- src/Ocelot.Provider.Kubernetes/Kube.cs | 34 ++++++++++--------- test/Ocelot.UnitTests/Kubernetes/KubeTests.cs | 6 ++-- 2 files changed, 21 insertions(+), 19 deletions(-) diff --git a/src/Ocelot.Provider.Kubernetes/Kube.cs b/src/Ocelot.Provider.Kubernetes/Kube.cs index 511ad93f8..bc1942ed2 100644 --- a/src/Ocelot.Provider.Kubernetes/Kube.cs +++ b/src/Ocelot.Provider.Kubernetes/Kube.cs @@ -48,38 +48,40 @@ public virtual async Task> GetAsync() .ToList(); } + private string Message(string details) + => $"Failed to retrieve {EndPointsKubeKind.ResourceApiVersion}/{EndPointsKubeKind.ResourceKind} '{_configuration.KeyOfServiceInK8s}' in namespace '{_configuration.KubeNamespace}': {details}"; + private async Task GetEndpoint() { - string serviceName = _configuration.KeyOfServiceInK8s; - string kubeNamespace = _configuration.KubeNamespace; - try { return await _kubeApi .ResourceClient(client => new EndPointClientV1(client)) - .GetAsync(serviceName, kubeNamespace); + .GetAsync(_configuration.KeyOfServiceInK8s, _configuration.KubeNamespace); } - catch (KubeApiException kubeApiError) + catch (KubeApiException ex) { - _logger.LogError(() => + string Msg() { - StatusV1 errorResponse = kubeApiError.Status; - string httpStatusCode = "Unknown"; - if (kubeApiError.InnerException is HttpRequestException httpRequestError) + StatusV1 status = ex.Status; + string httpStatusCode = "-"; // Unknown + if (ex.InnerException is HttpRequestException e) { - httpStatusCode = httpRequestError.StatusCode.ToString(); + httpStatusCode = e.StatusCode.ToString(); } - return $"Failed to retrieve {EndPointsKubeKind.ResourceApiVersion}/{EndPointsKubeKind.ResourceKind} '{serviceName}' in namespace '{kubeNamespace}' (HTTP.{httpStatusCode}/{errorResponse.Status}/{errorResponse.Reason}): {errorResponse.Message}"; - }, kubeApiError); + return Message($"(HTTP.{httpStatusCode}/{status.Status}/{status.Reason}): {status.Message}"); + } + + _logger.LogError(Msg, ex); } - catch (HttpRequestException unexpectedRequestError) + catch (HttpRequestException ex) { - _logger.LogError(() => $"Failed to retrieve {EndPointsKubeKind.ResourceApiVersion}/{EndPointsKubeKind.ResourceKind} '{serviceName}' in namespace '{kubeNamespace}' ({unexpectedRequestError.HttpRequestError}/HTTP.{unexpectedRequestError.StatusCode}).", unexpectedRequestError); + _logger.LogError(() => Message($"({ex.HttpRequestError}/HTTP.{ex.StatusCode})."), ex); } - catch (Exception unexpectedError) + catch (Exception unexpected) { - _logger.LogError(() => $"Failed to retrieve {EndPointsKubeKind.ResourceApiVersion}/{EndPointsKubeKind.ResourceKind} '{serviceName}' in namespace '{kubeNamespace}' (an unexpected error occurred).", unexpectedError); + _logger.LogError(() => Message($"(an unexpected ex occurred)."), unexpected); } return null; diff --git a/test/Ocelot.UnitTests/Kubernetes/KubeTests.cs b/test/Ocelot.UnitTests/Kubernetes/KubeTests.cs index 0eae1eee3..57e9b2f0d 100644 --- a/test/Ocelot.UnitTests/Kubernetes/KubeTests.cs +++ b/test/Ocelot.UnitTests/Kubernetes/KubeTests.cs @@ -61,13 +61,13 @@ public async Task Should_return_service_from_k8s() [InlineData(HttpStatusCode.Forbidden)] [InlineData(HttpStatusCode.InternalServerError)] [InlineData(HttpStatusCode.NotFound)] - [Trait("Feat", "345")] + [Trait("PR", "2266")] public async Task Should_not_return_service_from_k8s_when_k8s_api_returns_error_response(HttpStatusCode expectedStatusCode) { // Arrange var given = GivenClientAndProvider(out var serviceBuilder); serviceBuilder.Setup(x => x.BuildServices(It.IsAny(), It.IsAny())) - .Returns(new Service[] { new(nameof(Should_return_service_from_k8s), new("localhost", 80), string.Empty, string.Empty, Array.Empty()) }); + .Returns(new Service[] { new(nameof(Should_not_return_service_from_k8s_when_k8s_api_returns_error_response), new("localhost", 80), string.Empty, string.Empty, Array.Empty()) }); var endpoints = GivenEndpoints(); using var kubernetes = GivenThereIsAFakeKubeServiceDiscoveryProvider( @@ -79,7 +79,7 @@ public async Task Should_not_return_service_from_k8s_when_k8s_api_returns_error_ out Lazy receivedToken); string expectedKubeApiErrorMessage = GetKubeApiErrorMessage(serviceName: given.ProviderOptions.KeyOfServiceInK8s, given.ProviderOptions.KubeNamespace, expectedStatusCode); - string expectedLogMessage = $"Failed to retrieve v1/Endpoints '{given.ProviderOptions.KeyOfServiceInK8s}' in namespace '{given.ProviderOptions.KubeNamespace}' (HTTP.{expectedStatusCode}/Failure/{expectedStatusCode}): {expectedKubeApiErrorMessage}"; + string expectedLogMessage = $"Failed to retrieve v1/Endpoints '{given.ProviderOptions.KeyOfServiceInK8s}' in namespace '{given.ProviderOptions.KubeNamespace}': (HTTP.{expectedStatusCode}/Failure/{expectedStatusCode}): {expectedKubeApiErrorMessage}"; _logger.Setup(logger => logger.LogError(It.IsAny>(), It.IsAny())) .Callback((Func messageFactory, Exception exception) => { From 9b4e5cd40d465610e510912dd1cfe801ab202141 Mon Sep 17 00:00:00 2001 From: Adam Friedman Date: Tue, 18 Mar 2025 10:39:54 +1100 Subject: [PATCH 10/11] Change EndPointClientV1.GetAsync into a non-async wrapper method, since it only has a single await as the last statement (return await) --- src/Ocelot.Provider.Kubernetes/EndPointClientV1.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Ocelot.Provider.Kubernetes/EndPointClientV1.cs b/src/Ocelot.Provider.Kubernetes/EndPointClientV1.cs index e94c1a3d1..40c5fcff1 100644 --- a/src/Ocelot.Provider.Kubernetes/EndPointClientV1.cs +++ b/src/Ocelot.Provider.Kubernetes/EndPointClientV1.cs @@ -13,7 +13,7 @@ public EndPointClientV1(IKubeApiClient client) : base(client) { } - public async Task GetAsync(string serviceName, string kubeNamespace = null, CancellationToken cancellationToken = default) + public Task GetAsync(string serviceName, string kubeNamespace = null, CancellationToken cancellationToken = default) { if (string.IsNullOrEmpty(serviceName)) { @@ -27,7 +27,7 @@ public async Task GetAsync(string serviceName, string kubeNamespace ServiceName = serviceName, }); - return await Http.GetAsync(request, cancellationToken) + return Http.GetAsync(request, cancellationToken) .ReadContentAsObjectV1Async(operationDescription: $"get {nameof(EndpointsV1)}"); } } From 71c36e41d65b749849f5a319d4706ea93cd52d6f Mon Sep 17 00:00:00 2001 From: Raman Maksimchuk Date: Tue, 18 Mar 2025 21:34:53 +0300 Subject: [PATCH 11/11] Update docs with user's instructions --- docs/features/kubernetes.rst | 29 ++++++++++++++++++++---- samples/Kubernetes/ApiGateway/Program.cs | 19 +++++++++++++--- 2 files changed, 41 insertions(+), 7 deletions(-) diff --git a/docs/features/kubernetes.rst b/docs/features/kubernetes.rst index d7d9b71a7..3c155542c 100644 --- a/docs/features/kubernetes.rst +++ b/docs/features/kubernetes.rst @@ -36,7 +36,7 @@ The first thing you need to do is install the `package`_ that provides |kubernet Install-Package Ocelot.Provider.Kubernetes ``AddKubernetes(bool)`` method -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +------------------------------ .. code-block:: csharp :emphasize-lines: 3 @@ -84,19 +84,40 @@ If you have services deployed in Kubernetes, you will normally use the naming se }; builder.Services .AddOptions() - .Configure(configureKubeClient); // mannual binding options via IOptions + .Configure(configureKubeClient); // manual binding options via IOptions builder.Services .AddOcelot(builder.Configuration) .AddKubernetes(false); // don't use pod service account, and IOptions is reused + .. _break: http://break.do + + **Note**, this could also be written like this (shortened version): + + .. code-block:: csharp + :emphasize-lines: 2, 10 + + builder.Services + .AddKubeClientOptions(opts => + { + opts.ApiEndPoint = new UriBuilder("https", "my-host", 443).Uri; + opts.AuthStrategy = KubeAuthStrategy.BearerToken; + opts.AccessToken = "my-token"; + opts.AllowInsecure = true; + }) + .AddOcelot(builder.Configuration) + .AddKubernetes(false); // don't use pod service account, and client options provided via AddKubeClientOptions + Finally, it creates the `KubeClient`_ from your options. - **Note**: For understanding the ``IOptions`` interface, please refer to the Microsoft Learn documentation: `Options pattern in .NET `_. + **Note 1**: For understanding the ``IOptions`` interface, please refer to the Microsoft Learn documentation: `Options pattern in .NET `_. + + **Note 2**: Please consider this Case 2 as an example of manual setup when you **do not** use a pod service account. + We recommend using our official extension method, which receives an ``Action`` argument with your options: refer to the :ref:`k8s-addkubernetes-action-method` below. .. _k8s-addkubernetes-action-method: ``AddKubernetes(Action)`` method [#f2]_ -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +---------------------------------------------------------- .. code-block:: csharp :emphasize-lines: 3 diff --git a/samples/Kubernetes/ApiGateway/Program.cs b/samples/Kubernetes/ApiGateway/Program.cs index ba9c98f4b..988727900 100644 --- a/samples/Kubernetes/ApiGateway/Program.cs +++ b/samples/Kubernetes/ApiGateway/Program.cs @@ -10,7 +10,7 @@ .SetBasePath(builder.Environment.ContentRootPath) .AddOcelot(); -goto Case4; // Your case should be selected here!!! +goto Case5; // Your case should be selected here!!! // Link: https://github.com/ThreeMammals/Ocelot/blob/develop/docs/features/kubernetes.rst#addkubernetes-bool-method Case1: // Use a pod service account @@ -36,7 +36,7 @@ goto Start; // Link: https://github.com/ThreeMammals/Ocelot/blob/develop/docs/features/kubernetes.rst#addkubernetes-action-kubeclientoptions-method -Case3: // Use global ServiceDiscoveryProvider json-options +Case3: // Don't use a pod service account, manually bind options, ignore global ServiceDiscoveryProvider json-options Action myOptions = opts => { opts.ApiEndPoint = new UriBuilder(Uri.UriSchemeHttps, "my-host", 443).Uri; @@ -49,8 +49,21 @@ .AddKubernetes(myOptions); // configure options with action, without optional args goto Start; +Case4: // Don't use a pod service account, manually bind options, ignore global ServiceDiscoveryProvider json-options +builder.Services + .AddKubeClientOptions(opts => + { + opts.ApiEndPoint = new UriBuilder("https", "my-host", 443).Uri; + opts.AuthStrategy = KubeAuthStrategy.BearerToken; + opts.AccessToken = "my-token"; + opts.AllowInsecure = true; + }) + .AddOcelot(builder.Configuration) + .AddKubernetes(false); // don't use pod service account, and client options provided via AddKubeClientOptions +goto Start; + // Link: https://github.com/ThreeMammals/Ocelot/blob/develop/docs/features/kubernetes.rst#addkubernetes-action-kubeclientoptions-method -Case4: // Use global ServiceDiscoveryProvider json-options +Case5: // Use global ServiceDiscoveryProvider json-options Action? none = null; builder.Services .AddOcelot(builder.Configuration)