From c9c048bd55670f2c3419789d0ddda66b11eb572f Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Wed, 29 Apr 2026 14:21:03 +1000 Subject: [PATCH 01/14] Support Gateway TLS without pre-known hostname (FQDN discovery) When WithTls() is called without WithHostname(), the Gateway now: 1. Generates an HTTPS listener without a hostname restriction 2. After Helm deploy, polls Gateway status for the assigned address 3. Patches the HTTPS listener to add the discovered hostname 4. Creates a bootstrap self-signed TLS secret with the discovered FQDN 5. cert-manager then detects the hostname and issues a real certificate This enables a single-deploy TLS workflow for controllers like AGC that assign FQDNs automatically (e.g., *.alb.azure.com), without requiring users to deploy once to discover the FQDN and then redeploy with it. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../KubernetesEnvironmentResource.cs | 349 +++++++++++++++++- .../KubernetesGatewayTests.cs | 42 +++ 2 files changed, 386 insertions(+), 5 deletions(-) diff --git a/src/Aspire.Hosting.Kubernetes/KubernetesEnvironmentResource.cs b/src/Aspire.Hosting.Kubernetes/KubernetesEnvironmentResource.cs index a17d583828f..2c8551ea26c 100644 --- a/src/Aspire.Hosting.Kubernetes/KubernetesEnvironmentResource.cs +++ b/src/Aspire.Hosting.Kubernetes/KubernetesEnvironmentResource.cs @@ -238,6 +238,27 @@ public KubernetesEnvironmentResource(string name) : base(name) steps.Add(tlsBootstrapStep); } + // FQDN discovery step — for Gateway TLS configs with no hostnames, waits for the + // Gateway to be assigned an address, patches the listener hostname, and bootstraps TLS. + var gatewaysNeedingDiscovery = CollectGatewaysNeedingFqdnDiscovery(model, environment); + if (gatewaysNeedingDiscovery.Count > 0) + { + var fqdnDiscoveryStep = new PipelineStep + { + Name = $"tls-fqdn-discovery-{environment.Name}", + Description = "Discovers Gateway FQDN, patches listener hostname, and bootstraps TLS", + Action = ctx => DiscoverFqdnAndBootstrapTlsAsync(ctx, environment, gatewaysNeedingDiscovery) + }; + fqdnDiscoveryStep.DependsOn($"helm-deploy-{environment.Name}"); + if (tlsSecrets.Count > 0) + { + // Run after normal TLS bootstrap (which handles hostnames that are already known) + fqdnDiscoveryStep.DependsOn($"tls-bootstrap-{environment.Name}"); + } + fqdnDiscoveryStep.RequiredBy(WellKnownPipelineSteps.Deploy); + steps.Add(fqdnDiscoveryStep); + } + // Expand deployment target steps for compute resources (including dashboard if enabled) var resources = environment.DashboardEnabled && environment.Dashboard?.Resource is KubernetesAspireDashboardResource dashboard ? [.. model.GetComputeResources(), dashboard] @@ -718,20 +739,21 @@ private static async Task BuildGatewayObjects( var tlsListenerIndex = 0; foreach (var tls in gatewayResource.TlsConfigs) { - foreach (var host in tls.Hosts) + var resolvedSecretName = await ResolveExpressionAsync(tls.SecretName, cancellationToken).ConfigureAwait(false); + + if (tls.Hosts.Count == 0) { + // No hostnames specified — create an HTTPS listener without a hostname restriction. + // The hostname will be discovered from the Gateway's assigned address after deployment + // and patched onto the listener to enable cert-manager certificate issuance. var listenerName = tlsListenerIndex == 0 ? "https" : $"https-{tlsListenerIndex}"; tlsListenerIndex++; - var resolvedHost = await ResolveExpressionAsync(host, cancellationToken).ConfigureAwait(false); - var resolvedSecretName = await ResolveExpressionAsync(tls.SecretName, cancellationToken).ConfigureAwait(false); - gateway.Spec.Listeners.Add(new GatewayListenerV1 { Name = listenerName, Protocol = "HTTPS", Port = 443, - Hostname = resolvedHost, Tls = new GatewayTlsConfigV1 { Mode = "Terminate", @@ -743,6 +765,33 @@ private static async Task BuildGatewayObjects( } }); } + else + { + foreach (var host in tls.Hosts) + { + var listenerName = tlsListenerIndex == 0 ? "https" : $"https-{tlsListenerIndex}"; + tlsListenerIndex++; + + var resolvedHost = await ResolveExpressionAsync(host, cancellationToken).ConfigureAwait(false); + + gateway.Spec.Listeners.Add(new GatewayListenerV1 + { + Name = listenerName, + Protocol = "HTTPS", + Port = 443, + Hostname = resolvedHost, + Tls = new GatewayTlsConfigV1 + { + Mode = "Terminate", + CertificateRefs = { new GatewayCertificateRefV1 { Name = resolvedSecretName } } + }, + AllowedRoutes = new GatewayAllowedRoutesV1 + { + Namespaces = new GatewayRouteNamespacesV1 { From = "Same" } + } + }); + } + } } gatewayResource.GeneratedGateway = gateway; @@ -870,6 +919,296 @@ private static async Task BuildGatewayObjects( return tlsSecrets; } + /// + /// Collects gateway TLS configurations that have no hostnames specified and need + /// the assigned FQDN to be discovered from the Gateway's status after deployment. + /// + private static List<(KubernetesGatewayResource Gateway, ReferenceExpression SecretName)> CollectGatewaysNeedingFqdnDiscovery( + DistributedApplicationModel model, + KubernetesEnvironmentResource environment) + { + var results = new List<(KubernetesGatewayResource Gateway, ReferenceExpression SecretName)>(); + + foreach (var gateway in model.Resources.OfType().Where(g => g.Parent == environment)) + { + foreach (var tls in gateway.TlsConfigs) + { + if (tls.Hosts.Count == 0) + { + results.Add((gateway, tls.SecretName)); + } + } + } + + return results; + } + + /// + /// Discovers the assigned FQDN from the Gateway's status, patches the HTTPS listener + /// to include the hostname, and creates a bootstrap TLS secret so cert-manager can + /// issue a real certificate. + /// + private static async Task DiscoverFqdnAndBootstrapTlsAsync( + PipelineStepContext context, + KubernetesEnvironmentResource environment, + List<(KubernetesGatewayResource Gateway, ReferenceExpression SecretName)> gatewaysNeedingDiscovery) + { + var @namespace = "default"; + if (environment.TryGetLastAnnotation(out var nsAnnotation)) + { + var resolvedNs = await nsAnnotation.Namespace.GetValueAsync(context.CancellationToken).ConfigureAwait(false); + if (!string.IsNullOrEmpty(resolvedNs)) + { + @namespace = resolvedNs; + } + } + + foreach (var (gateway, secretNameExpr) in gatewaysNeedingDiscovery) + { + var gatewayName = gateway.Name.ToKubernetesResourceName(); + var secretName = await ResolveExpressionAsync(secretNameExpr, context.CancellationToken).ConfigureAwait(false); + + // Poll for the Gateway's assigned address (load balancer controllers may take time to provision) + context.Logger.LogInformation( + "Waiting for Gateway '{GatewayName}' to be assigned an address...", gatewayName); + + string? discoveredFqdn = null; + var maxAttempts = 60; // 5 minutes with 5s intervals + for (var attempt = 0; attempt < maxAttempts; attempt++) + { + var jsonPathArgs = $"get gateway {gatewayName} --namespace {@namespace} -o jsonpath=\"{{.status.addresses[0].value}}\""; + if (environment.KubeConfigPath is not null) + { + jsonPathArgs += $" --kubeconfig \"{environment.KubeConfigPath}\""; + } + + var stdout = new List(); + var (getResult, getDisposable) = ProcessUtil.Run(new ProcessSpec("kubectl") + { + Arguments = jsonPathArgs, + ThrowOnNonZeroReturnCode = false, + InheritEnv = true, + OnOutputData = line => + { + if (!string.IsNullOrWhiteSpace(line)) + { + stdout.Add(line); + } + }, + OnErrorData = _ => { } + }); + + await using (getDisposable.ConfigureAwait(false)) + { + var result = await getResult.WaitAsync(context.CancellationToken).ConfigureAwait(false); + if (result.ExitCode == 0 && stdout.Count > 0 && !string.IsNullOrWhiteSpace(stdout[0])) + { + discoveredFqdn = stdout[0].Trim(); + break; + } + } + + if (attempt < maxAttempts - 1) + { + await Task.Delay(TimeSpan.FromSeconds(5), context.CancellationToken).ConfigureAwait(false); + } + } + + if (string.IsNullOrEmpty(discoveredFqdn)) + { + context.Logger.LogWarning( + "Gateway '{GatewayName}' was not assigned an address after waiting. " + + "TLS hostname discovery skipped. You may need to redeploy with an explicit hostname via WithHostname().", + gatewayName); + continue; + } + + context.Logger.LogInformation( + "Gateway '{GatewayName}' assigned address: {Fqdn}. Patching HTTPS listener(s) and bootstrapping TLS.", + gatewayName, discoveredFqdn); + + // Find HTTPS listeners without a hostname and patch them with the discovered FQDN. + // We query the Gateway spec to find the correct listener indices dynamically. + var listenerJsonArgs = $"get gateway {gatewayName} --namespace {@namespace} -o jsonpath=\"{{.spec.listeners}}\""; + if (environment.KubeConfigPath is not null) + { + listenerJsonArgs += $" --kubeconfig \"{environment.KubeConfigPath}\""; + } + + var listenerJson = new List(); + var (listenerResult, listenerDisposable) = ProcessUtil.Run(new ProcessSpec("kubectl") + { + Arguments = listenerJsonArgs, + ThrowOnNonZeroReturnCode = false, + InheritEnv = true, + OnOutputData = line => + { + if (!string.IsNullOrWhiteSpace(line)) + { + listenerJson.Add(line); + } + }, + OnErrorData = _ => { } + }); + + var httpsListenerIndices = new List(); + await using (listenerDisposable.ConfigureAwait(false)) + { + var listenerExitResult = await listenerResult.WaitAsync(context.CancellationToken).ConfigureAwait(false); + if (listenerExitResult.ExitCode == 0 && listenerJson.Count > 0) + { + try + { + using var doc = System.Text.Json.JsonDocument.Parse(string.Join("", listenerJson)); + var listeners = doc.RootElement; + for (var i = 0; i < listeners.GetArrayLength(); i++) + { + var listener = listeners[i]; + if (listener.TryGetProperty("protocol", out var protocol) && + string.Equals(protocol.GetString(), "HTTPS", StringComparison.OrdinalIgnoreCase) && + !listener.TryGetProperty("hostname", out _)) + { + httpsListenerIndices.Add(i); + } + } + } + catch (System.Text.Json.JsonException) + { + // Fall back to assuming index 1 (HTTP=0, HTTPS=1) + httpsListenerIndices.Add(1); + } + } + else + { + // Fall back to assuming index 1 + httpsListenerIndices.Add(1); + } + } + + if (httpsListenerIndices.Count == 0) + { + context.Logger.LogWarning( + "No HTTPS listeners without hostname found on Gateway '{GatewayName}'. Skipping hostname patch.", + gatewayName); + } + else + { + // Build a JSON patch that adds the hostname to all HTTPS listeners that don't have one + var patchOps = string.Join(",", httpsListenerIndices.Select( + idx => $"{{\"op\":\"add\",\"path\":\"/spec/listeners/{idx}/hostname\",\"value\":\"{discoveredFqdn}\"}}")); + var patchJson = $"[{patchOps}]"; + + var patchArgs = $"patch gateway {gatewayName} --namespace {@namespace} --type=json -p=\"{patchJson.Replace("\"", "\\\"")}\""; + if (environment.KubeConfigPath is not null) + { + patchArgs += $" --kubeconfig \"{environment.KubeConfigPath}\""; + } + + var (patchResult, patchDisposable) = ProcessUtil.Run(new ProcessSpec("kubectl") + { + Arguments = patchArgs, + ThrowOnNonZeroReturnCode = false, + InheritEnv = true, + OnOutputData = line => context.Logger.LogDebug("{Line}", line), + OnErrorData = line => context.Logger.LogDebug("{Line}", line) + }); + + await using (patchDisposable.ConfigureAwait(false)) + { + var patchExitResult = await patchResult.WaitAsync(context.CancellationToken).ConfigureAwait(false); + if (patchExitResult.ExitCode != 0) + { + context.Logger.LogWarning( + "Failed to patch Gateway '{GatewayName}' with hostname '{Hostname}' (exit code {ExitCode}). " + + "You may need to redeploy with an explicit hostname via WithHostname().", + gatewayName, discoveredFqdn, patchExitResult.ExitCode); + continue; + } + } + } + + // Check if bootstrap TLS secret already exists + var checkArgs = $"get secret {secretName} --namespace {@namespace}"; + if (environment.KubeConfigPath is not null) + { + checkArgs += $" --kubeconfig \"{environment.KubeConfigPath}\""; + } + + var (checkResult, checkDisposable) = ProcessUtil.Run(new ProcessSpec("kubectl") + { + Arguments = checkArgs, + ThrowOnNonZeroReturnCode = false, + InheritEnv = true, + OnOutputData = _ => { }, + OnErrorData = _ => { } + }); + + await using (checkDisposable.ConfigureAwait(false)) + { + var result = await checkResult.WaitAsync(context.CancellationToken).ConfigureAwait(false); + if (result.ExitCode == 0) + { + context.Logger.LogInformation("TLS secret '{SecretName}' already exists, skipping bootstrap.", secretName); + continue; + } + } + + // Create a bootstrap self-signed cert with the discovered FQDN + context.Logger.LogInformation("Creating bootstrap TLS secret '{SecretName}' for '{Hostname}'.", secretName, discoveredFqdn); + + using var ecdsa = ECDsa.Create(ECCurve.NamedCurves.nistP256); + var certRequest = new CertificateRequest($"CN={discoveredFqdn}", ecdsa, HashAlgorithmName.SHA256); + using var cert = certRequest.CreateSelfSigned(DateTimeOffset.UtcNow, DateTimeOffset.UtcNow.AddDays(1)); + + var certPem = cert.ExportCertificatePem(); + var keyPem = ecdsa.ExportECPrivateKeyPem(); + + var tempDir = Directory.CreateTempSubdirectory(".aspire-tls-discovery"); + try + { + var certPath = Path.Combine(tempDir.FullName, "tls.crt"); + var keyPath = Path.Combine(tempDir.FullName, "tls.key"); + await File.WriteAllTextAsync(certPath, certPem, context.CancellationToken).ConfigureAwait(false); + await File.WriteAllTextAsync(keyPath, keyPem, context.CancellationToken).ConfigureAwait(false); + + var createArgs = $"create secret tls {secretName} --cert=\"{certPath}\" --key=\"{keyPath}\" --namespace {@namespace}"; + if (environment.KubeConfigPath is not null) + { + createArgs += $" --kubeconfig \"{environment.KubeConfigPath}\""; + } + + var (createResult, createDisposable) = ProcessUtil.Run(new ProcessSpec("kubectl") + { + Arguments = createArgs, + ThrowOnNonZeroReturnCode = false, + InheritEnv = true, + OnOutputData = line => context.Logger.LogDebug("{Line}", line), + OnErrorData = line => context.Logger.LogDebug("{Line}", line) + }); + + await using (createDisposable.ConfigureAwait(false)) + { + var createExitResult = await createResult.WaitAsync(context.CancellationToken).ConfigureAwait(false); + if (createExitResult.ExitCode != 0) + { + context.Logger.LogWarning("Failed to create bootstrap TLS secret '{SecretName}' (exit code {ExitCode}).", secretName, createExitResult.ExitCode); + } + else + { + context.Logger.LogInformation( + "Bootstrap TLS secret '{SecretName}' created for '{Hostname}'. " + + "cert-manager will replace this with a real certificate once the hostname is detected on the Gateway listener.", + secretName, discoveredFqdn); + } + } + } + finally + { + try { tempDir.Delete(recursive: true); } catch { } + } + } + } + private static async Task BootstrapTlsSecretsAsync( PipelineStepContext context, KubernetesEnvironmentResource environment, diff --git a/tests/Aspire.Hosting.Kubernetes.Tests/KubernetesGatewayTests.cs b/tests/Aspire.Hosting.Kubernetes.Tests/KubernetesGatewayTests.cs index 88073a59ac7..99d0c227aa7 100644 --- a/tests/Aspire.Hosting.Kubernetes.Tests/KubernetesGatewayTests.cs +++ b/tests/Aspire.Hosting.Kubernetes.Tests/KubernetesGatewayTests.cs @@ -227,4 +227,46 @@ public async Task AddGateway_BackwardCompatible_NoGatewayNoChange() Assert.DoesNotContain(files, f => f.Contains("gateway", StringComparison.OrdinalIgnoreCase)); Assert.DoesNotContain(files, f => f.Contains("route", StringComparison.OrdinalIgnoreCase)); } + + [Fact] + public async Task AddGateway_WithTls_NoHostname_GeneratesHttpsListenerWithoutHostname() + { + using var tempDir = new TestTempDirectory(); + var builder = TestDistributedApplicationBuilder.Create(DistributedApplicationOperation.Publish, tempDir.Path); + + var k8s = builder.AddKubernetesEnvironment("env"); + var gateway = k8s.AddGateway("public").WithGatewayClass("azure-alb-external"); + + var api = builder.AddContainer("myapi", "nginx") + .WithHttpEndpoint(targetPort: 8080); + + // WithTls() without WithHostname() — should still generate an HTTPS listener + gateway + .WithRoute("/", api.GetEndpoint("http")) + .WithTls("my-tls-secret"); + + var app = builder.Build(); + app.Run(); + + // Check Gateway has HTTPS listener without a hostname + var gatewayFile = Path.Combine(tempDir.Path, "templates", "public", "public.yaml"); + var content = await File.ReadAllTextAsync(gatewayFile); + + Assert.Contains("HTTPS", content); + Assert.Contains("Terminate", content); + Assert.Contains("my-tls-secret", content); + // Should also have HTTP listener + Assert.Contains("HTTP", content); + + // The HTTPS listener should NOT have a hostname field (since no WithHostname was called) + // Verify it has the listener but the hostname line should not appear after HTTPS + var lines = content.Split('\n').Select(l => l.Trim()).ToList(); + var httpsIndex = lines.FindIndex(l => l.Contains("protocol: HTTPS")); + Assert.True(httpsIndex >= 0, "HTTPS listener not found"); + + // Find the next listener or end of listeners to check there's no hostname + var nextListenerOrEnd = lines.FindIndex(httpsIndex + 1, l => l.StartsWith("- name:") || l == ""); + var httpsSection = lines.Skip(httpsIndex).Take((nextListenerOrEnd > httpsIndex ? nextListenerOrEnd : lines.Count) - httpsIndex); + Assert.DoesNotContain(httpsSection, l => l.StartsWith("hostname:")); + } } From c4f6ec66a49a09d7263ee36efba242ea897d5deb Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Wed, 29 Apr 2026 15:11:24 +1000 Subject: [PATCH 02/14] Use helm field-manager for Gateway hostname patch to avoid conflicts After patching the Gateway hostname via JSON patch, re-apply the full Gateway YAML with --server-side --field-manager=helm --force-conflicts to transfer field ownership back to Helm. This prevents SSA conflicts when the user later redeploys with an explicit hostname via WithHostname(). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../KubernetesEnvironmentResource.cs | 66 ++++++++++++++++++- 1 file changed, 65 insertions(+), 1 deletion(-) diff --git a/src/Aspire.Hosting.Kubernetes/KubernetesEnvironmentResource.cs b/src/Aspire.Hosting.Kubernetes/KubernetesEnvironmentResource.cs index 2c8551ea26c..1881facf359 100644 --- a/src/Aspire.Hosting.Kubernetes/KubernetesEnvironmentResource.cs +++ b/src/Aspire.Hosting.Kubernetes/KubernetesEnvironmentResource.cs @@ -1093,7 +1093,9 @@ private static async Task DiscoverFqdnAndBootstrapTlsAsync( } else { - // Build a JSON patch that adds the hostname to all HTTPS listeners that don't have one + // Patch the HTTPS listeners with the discovered hostname using JSON patch, + // then transfer field ownership to Helm via server-side apply so that + // subsequent Helm deploys don't encounter SSA conflicts. var patchOps = string.Join(",", httpsListenerIndices.Select( idx => $"{{\"op\":\"add\",\"path\":\"/spec/listeners/{idx}/hostname\",\"value\":\"{discoveredFqdn}\"}}")); var patchJson = $"[{patchOps}]"; @@ -1125,6 +1127,68 @@ private static async Task DiscoverFqdnAndBootstrapTlsAsync( continue; } } + + // Transfer field ownership to Helm using server-side apply so subsequent + // Helm deploys don't encounter SSA conflicts on the patched hostname field. + var currentGatewayArgs = $"get gateway {gatewayName} --namespace {@namespace} -o yaml"; + if (environment.KubeConfigPath is not null) + { + currentGatewayArgs += $" --kubeconfig \"{environment.KubeConfigPath}\""; + } + + var gatewayYamlLines = new List(); + var (getGwResult, getGwDisposable) = ProcessUtil.Run(new ProcessSpec("kubectl") + { + Arguments = currentGatewayArgs, + ThrowOnNonZeroReturnCode = false, + InheritEnv = true, + OnOutputData = gatewayYamlLines.Add, + OnErrorData = _ => { } + }); + + await using (getGwDisposable.ConfigureAwait(false)) + { + var getGwExitResult = await getGwResult.WaitAsync(context.CancellationToken).ConfigureAwait(false); + if (getGwExitResult.ExitCode == 0 && gatewayYamlLines.Count > 0) + { + var tempFile = Path.GetTempFileName(); + try + { + await File.WriteAllLinesAsync(tempFile, gatewayYamlLines, context.CancellationToken).ConfigureAwait(false); + + var applyArgs = $"apply --server-side --field-manager=helm --force-conflicts -f \"{tempFile}\""; + if (environment.KubeConfigPath is not null) + { + applyArgs += $" --kubeconfig \"{environment.KubeConfigPath}\""; + } + + var (applyResult, applyDisposable) = ProcessUtil.Run(new ProcessSpec("kubectl") + { + Arguments = applyArgs, + ThrowOnNonZeroReturnCode = false, + InheritEnv = true, + OnOutputData = line => context.Logger.LogDebug("{Line}", line), + OnErrorData = line => context.Logger.LogDebug("{Line}", line) + }); + + await using (applyDisposable.ConfigureAwait(false)) + { + var applyExitResult = await applyResult.WaitAsync(context.CancellationToken).ConfigureAwait(false); + if (applyExitResult.ExitCode != 0) + { + context.Logger.LogDebug( + "Failed to transfer field ownership to Helm (exit code {ExitCode}). " + + "Subsequent deploys with an explicit hostname may require --force.", + applyExitResult.ExitCode); + } + } + } + finally + { + try { File.Delete(tempFile); } catch { } + } + } + } } // Check if bootstrap TLS secret already exists From 0aba6af4461e7d14315b46e3a92a09bebcaa3f79 Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Wed, 29 Apr 2026 15:55:04 +1000 Subject: [PATCH 03/14] Fix endpoint resource lookup for TS AppHost RPC bridge Use ResourceNameComparer on the deploymentTargets dictionary so that endpoint references created through the TypeScript AppHost RPC bridge (which may use a different resource instance) resolve correctly by resource name. This matches the pattern already used in KubernetesEnvironmentContext._kubernetesComponents. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../KubernetesEnvironmentResource.cs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/Aspire.Hosting.Kubernetes/KubernetesEnvironmentResource.cs b/src/Aspire.Hosting.Kubernetes/KubernetesEnvironmentResource.cs index 1881facf359..50375dc1e0d 100644 --- a/src/Aspire.Hosting.Kubernetes/KubernetesEnvironmentResource.cs +++ b/src/Aspire.Hosting.Kubernetes/KubernetesEnvironmentResource.cs @@ -385,8 +385,11 @@ private async Task PrepareDeploymentTargetsAsync(PipelineStepContext context) }); } - // Build deployment target lookup for endpoint resolution - var deploymentTargets = new Dictionary(); + // Build deployment target lookup for endpoint resolution. + // Use name-based equality so that endpoint references created through the + // TypeScript AppHost RPC bridge (which may use a different resource instance) + // resolve correctly. + var deploymentTargets = new Dictionary(new ResourceNameComparer()); foreach (var r in appModel.GetComputeResources()) { From 03c70e8b4b8d984fc5c6ea63b23f7dab78460107 Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Wed, 29 Apr 2026 17:34:55 +1000 Subject: [PATCH 04/14] Fix test assertion for YAML quoted protocol values The YAML serializer quotes string values like protocol: "HTTPS" on CI. Use a more flexible assertion that matches both quoted and unquoted forms. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../KubernetesGatewayTests.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/Aspire.Hosting.Kubernetes.Tests/KubernetesGatewayTests.cs b/tests/Aspire.Hosting.Kubernetes.Tests/KubernetesGatewayTests.cs index 99d0c227aa7..bc3186aed79 100644 --- a/tests/Aspire.Hosting.Kubernetes.Tests/KubernetesGatewayTests.cs +++ b/tests/Aspire.Hosting.Kubernetes.Tests/KubernetesGatewayTests.cs @@ -261,12 +261,12 @@ public async Task AddGateway_WithTls_NoHostname_GeneratesHttpsListenerWithoutHos // The HTTPS listener should NOT have a hostname field (since no WithHostname was called) // Verify it has the listener but the hostname line should not appear after HTTPS var lines = content.Split('\n').Select(l => l.Trim()).ToList(); - var httpsIndex = lines.FindIndex(l => l.Contains("protocol: HTTPS")); - Assert.True(httpsIndex >= 0, "HTTPS listener not found"); + var httpsIndex = lines.FindIndex(l => l.Contains("protocol:") && l.Contains("HTTPS")); + Assert.True(httpsIndex >= 0, "HTTPS listener not found in:\n" + content); // Find the next listener or end of listeners to check there's no hostname var nextListenerOrEnd = lines.FindIndex(httpsIndex + 1, l => l.StartsWith("- name:") || l == ""); var httpsSection = lines.Skip(httpsIndex).Take((nextListenerOrEnd > httpsIndex ? nextListenerOrEnd : lines.Count) - httpsIndex); - Assert.DoesNotContain(httpsSection, l => l.StartsWith("hostname:")); + Assert.DoesNotContain(httpsSection, l => l.StartsWith("hostname:") || l.StartsWith("hostname ")); } } From ba3872113085bdbe93279071d8a3de802ca6f054 Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Wed, 29 Apr 2026 19:34:47 +1000 Subject: [PATCH 05/14] Address all automated review feedback 1. Address type validation: Parse full Gateway JSON status, prefer Hostname-type addresses, fall back to DNS-like values, skip IPs. 2. Use -o json instead of jsonpath: Parse full Gateway JSON for both address discovery and listener index detection. More reliable. 3. Use JsonSerializer + --patch-file: Build JSON patch operations with proper serialization, write to temp file to avoid shell escaping. 4. Minimal manifest for field ownership: Read current Gateway JSON, strip server fields (status, resourceVersion, managedFields), keep only apiVersion/kind/metadata(name,namespace)/spec for SSA apply. 5. Temp file pattern: Use CreateTempSubdirectory consistently. 6. Add SAN to bootstrap certs: Add SubjectAlternativeNameBuilder with DNS name in both DiscoverFqdnAndBootstrapTlsAsync and the existing BootstrapTlsSecretsAsync. Modern TLS clients require SAN. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../KubernetesEnvironmentResource.cs | 443 ++++++++++++------ 1 file changed, 287 insertions(+), 156 deletions(-) diff --git a/src/Aspire.Hosting.Kubernetes/KubernetesEnvironmentResource.cs b/src/Aspire.Hosting.Kubernetes/KubernetesEnvironmentResource.cs index 50375dc1e0d..d4e3887333d 100644 --- a/src/Aspire.Hosting.Kubernetes/KubernetesEnvironmentResource.cs +++ b/src/Aspire.Hosting.Kubernetes/KubernetesEnvironmentResource.cs @@ -971,43 +971,42 @@ private static async Task DiscoverFqdnAndBootstrapTlsAsync( var gatewayName = gateway.Name.ToKubernetesResourceName(); var secretName = await ResolveExpressionAsync(secretNameExpr, context.CancellationToken).ConfigureAwait(false); - // Poll for the Gateway's assigned address (load balancer controllers may take time to provision) + // Poll for the Gateway's assigned hostname address. + // We use -o json and parse the full status to select Hostname-type addresses, + // since some controllers return IP addresses which are not valid for TLS hostnames. context.Logger.LogInformation( - "Waiting for Gateway '{GatewayName}' to be assigned an address...", gatewayName); + "Waiting for Gateway '{GatewayName}' to be assigned a hostname address...", gatewayName); string? discoveredFqdn = null; var maxAttempts = 60; // 5 minutes with 5s intervals for (var attempt = 0; attempt < maxAttempts; attempt++) { - var jsonPathArgs = $"get gateway {gatewayName} --namespace {@namespace} -o jsonpath=\"{{.status.addresses[0].value}}\""; + var getArgs = $"get gateway {gatewayName} --namespace {@namespace} -o json"; if (environment.KubeConfigPath is not null) { - jsonPathArgs += $" --kubeconfig \"{environment.KubeConfigPath}\""; + getArgs += $" --kubeconfig \"{environment.KubeConfigPath}\""; } var stdout = new List(); var (getResult, getDisposable) = ProcessUtil.Run(new ProcessSpec("kubectl") { - Arguments = jsonPathArgs, + Arguments = getArgs, ThrowOnNonZeroReturnCode = false, InheritEnv = true, - OnOutputData = line => - { - if (!string.IsNullOrWhiteSpace(line)) - { - stdout.Add(line); - } - }, + OnOutputData = stdout.Add, OnErrorData = _ => { } }); await using (getDisposable.ConfigureAwait(false)) { var result = await getResult.WaitAsync(context.CancellationToken).ConfigureAwait(false); - if (result.ExitCode == 0 && stdout.Count > 0 && !string.IsNullOrWhiteSpace(stdout[0])) + if (result.ExitCode == 0 && stdout.Count > 0) { - discoveredFqdn = stdout[0].Trim(); - break; + discoveredFqdn = ExtractHostnameFromGatewayJson(string.Join("", stdout)); + if (discoveredFqdn is not null) + { + break; + } } } @@ -1020,7 +1019,7 @@ private static async Task DiscoverFqdnAndBootstrapTlsAsync( if (string.IsNullOrEmpty(discoveredFqdn)) { context.Logger.LogWarning( - "Gateway '{GatewayName}' was not assigned an address after waiting. " + + "Gateway '{GatewayName}' was not assigned a hostname address after waiting. " + "TLS hostname discovery skipped. You may need to redeploy with an explicit hostname via WithHostname().", gatewayName); continue; @@ -1030,63 +1029,9 @@ private static async Task DiscoverFqdnAndBootstrapTlsAsync( "Gateway '{GatewayName}' assigned address: {Fqdn}. Patching HTTPS listener(s) and bootstrapping TLS.", gatewayName, discoveredFqdn); - // Find HTTPS listeners without a hostname and patch them with the discovered FQDN. - // We query the Gateway spec to find the correct listener indices dynamically. - var listenerJsonArgs = $"get gateway {gatewayName} --namespace {@namespace} -o jsonpath=\"{{.spec.listeners}}\""; - if (environment.KubeConfigPath is not null) - { - listenerJsonArgs += $" --kubeconfig \"{environment.KubeConfigPath}\""; - } - - var listenerJson = new List(); - var (listenerResult, listenerDisposable) = ProcessUtil.Run(new ProcessSpec("kubectl") - { - Arguments = listenerJsonArgs, - ThrowOnNonZeroReturnCode = false, - InheritEnv = true, - OnOutputData = line => - { - if (!string.IsNullOrWhiteSpace(line)) - { - listenerJson.Add(line); - } - }, - OnErrorData = _ => { } - }); - - var httpsListenerIndices = new List(); - await using (listenerDisposable.ConfigureAwait(false)) - { - var listenerExitResult = await listenerResult.WaitAsync(context.CancellationToken).ConfigureAwait(false); - if (listenerExitResult.ExitCode == 0 && listenerJson.Count > 0) - { - try - { - using var doc = System.Text.Json.JsonDocument.Parse(string.Join("", listenerJson)); - var listeners = doc.RootElement; - for (var i = 0; i < listeners.GetArrayLength(); i++) - { - var listener = listeners[i]; - if (listener.TryGetProperty("protocol", out var protocol) && - string.Equals(protocol.GetString(), "HTTPS", StringComparison.OrdinalIgnoreCase) && - !listener.TryGetProperty("hostname", out _)) - { - httpsListenerIndices.Add(i); - } - } - } - catch (System.Text.Json.JsonException) - { - // Fall back to assuming index 1 (HTTP=0, HTTPS=1) - httpsListenerIndices.Add(1); - } - } - else - { - // Fall back to assuming index 1 - httpsListenerIndices.Add(1); - } - } + // Find HTTPS listeners without a hostname by parsing the full Gateway JSON. + var httpsListenerIndices = await FindHostnamelessHttpsListeners( + gatewayName, @namespace, environment, context).ConfigureAwait(false); if (httpsListenerIndices.Count == 0) { @@ -1096,102 +1041,61 @@ private static async Task DiscoverFqdnAndBootstrapTlsAsync( } else { - // Patch the HTTPS listeners with the discovered hostname using JSON patch, - // then transfer field ownership to Helm via server-side apply so that - // subsequent Helm deploys don't encounter SSA conflicts. - var patchOps = string.Join(",", httpsListenerIndices.Select( - idx => $"{{\"op\":\"add\",\"path\":\"/spec/listeners/{idx}/hostname\",\"value\":\"{discoveredFqdn}\"}}")); - var patchJson = $"[{patchOps}]"; - - var patchArgs = $"patch gateway {gatewayName} --namespace {@namespace} --type=json -p=\"{patchJson.Replace("\"", "\\\"")}\""; - if (environment.KubeConfigPath is not null) - { - patchArgs += $" --kubeconfig \"{environment.KubeConfigPath}\""; - } - - var (patchResult, patchDisposable) = ProcessUtil.Run(new ProcessSpec("kubectl") + // Build the JSON patch using proper serialization to avoid injection issues. + var patchOperations = httpsListenerIndices.Select(idx => new { - Arguments = patchArgs, - ThrowOnNonZeroReturnCode = false, - InheritEnv = true, - OnOutputData = line => context.Logger.LogDebug("{Line}", line), - OnErrorData = line => context.Logger.LogDebug("{Line}", line) + op = "add", + path = $"/spec/listeners/{idx}/hostname", + value = discoveredFqdn }); + var patchJson = System.Text.Json.JsonSerializer.Serialize(patchOperations); - await using (patchDisposable.ConfigureAwait(false)) + // Write patch to a temp file to avoid shell escaping issues with kubectl -p + var patchTempDir = Directory.CreateTempSubdirectory(".aspire-gateway-patch"); + try { - var patchExitResult = await patchResult.WaitAsync(context.CancellationToken).ConfigureAwait(false); - if (patchExitResult.ExitCode != 0) + var patchFilePath = Path.Combine(patchTempDir.FullName, "patch.json"); + await File.WriteAllTextAsync(patchFilePath, patchJson, context.CancellationToken).ConfigureAwait(false); + + var patchArgs = $"patch gateway {gatewayName} --namespace {@namespace} --type=json --patch-file \"{patchFilePath}\""; + if (environment.KubeConfigPath is not null) { - context.Logger.LogWarning( - "Failed to patch Gateway '{GatewayName}' with hostname '{Hostname}' (exit code {ExitCode}). " + - "You may need to redeploy with an explicit hostname via WithHostname().", - gatewayName, discoveredFqdn, patchExitResult.ExitCode); - continue; + patchArgs += $" --kubeconfig \"{environment.KubeConfigPath}\""; } - } - // Transfer field ownership to Helm using server-side apply so subsequent - // Helm deploys don't encounter SSA conflicts on the patched hostname field. - var currentGatewayArgs = $"get gateway {gatewayName} --namespace {@namespace} -o yaml"; - if (environment.KubeConfigPath is not null) - { - currentGatewayArgs += $" --kubeconfig \"{environment.KubeConfigPath}\""; - } - - var gatewayYamlLines = new List(); - var (getGwResult, getGwDisposable) = ProcessUtil.Run(new ProcessSpec("kubectl") - { - Arguments = currentGatewayArgs, - ThrowOnNonZeroReturnCode = false, - InheritEnv = true, - OnOutputData = gatewayYamlLines.Add, - OnErrorData = _ => { } - }); - - await using (getGwDisposable.ConfigureAwait(false)) - { - var getGwExitResult = await getGwResult.WaitAsync(context.CancellationToken).ConfigureAwait(false); - if (getGwExitResult.ExitCode == 0 && gatewayYamlLines.Count > 0) + var (patchResult, patchDisposable) = ProcessUtil.Run(new ProcessSpec("kubectl") { - var tempFile = Path.GetTempFileName(); - try - { - await File.WriteAllLinesAsync(tempFile, gatewayYamlLines, context.CancellationToken).ConfigureAwait(false); - - var applyArgs = $"apply --server-side --field-manager=helm --force-conflicts -f \"{tempFile}\""; - if (environment.KubeConfigPath is not null) - { - applyArgs += $" --kubeconfig \"{environment.KubeConfigPath}\""; - } + Arguments = patchArgs, + ThrowOnNonZeroReturnCode = false, + InheritEnv = true, + OnOutputData = line => context.Logger.LogDebug("{Line}", line), + OnErrorData = line => context.Logger.LogDebug("{Line}", line) + }); - var (applyResult, applyDisposable) = ProcessUtil.Run(new ProcessSpec("kubectl") - { - Arguments = applyArgs, - ThrowOnNonZeroReturnCode = false, - InheritEnv = true, - OnOutputData = line => context.Logger.LogDebug("{Line}", line), - OnErrorData = line => context.Logger.LogDebug("{Line}", line) - }); - - await using (applyDisposable.ConfigureAwait(false)) - { - var applyExitResult = await applyResult.WaitAsync(context.CancellationToken).ConfigureAwait(false); - if (applyExitResult.ExitCode != 0) - { - context.Logger.LogDebug( - "Failed to transfer field ownership to Helm (exit code {ExitCode}). " + - "Subsequent deploys with an explicit hostname may require --force.", - applyExitResult.ExitCode); - } - } - } - finally + await using (patchDisposable.ConfigureAwait(false)) + { + var patchExitResult = await patchResult.WaitAsync(context.CancellationToken).ConfigureAwait(false); + if (patchExitResult.ExitCode != 0) { - try { File.Delete(tempFile); } catch { } + context.Logger.LogWarning( + "Failed to patch Gateway '{GatewayName}' with hostname '{Hostname}' (exit code {ExitCode}). " + + "You may need to redeploy with an explicit hostname via WithHostname().", + gatewayName, discoveredFqdn, patchExitResult.ExitCode); + continue; } } } + finally + { + try { patchTempDir.Delete(recursive: true); } catch { } + } + + // Transfer field ownership to Helm using server-side apply with a minimal + // Gateway manifest so subsequent Helm deploys don't encounter SSA conflicts. + // We construct a minimal spec rather than re-applying kubectl get output, + // which would include server-populated fields (status, resourceVersion, etc.). + await TransferGatewayFieldOwnership( + gatewayName, @namespace, environment, context).ConfigureAwait(false); } // Check if bootstrap TLS secret already exists @@ -1225,6 +1129,9 @@ private static async Task DiscoverFqdnAndBootstrapTlsAsync( using var ecdsa = ECDsa.Create(ECCurve.NamedCurves.nistP256); var certRequest = new CertificateRequest($"CN={discoveredFqdn}", ecdsa, HashAlgorithmName.SHA256); + var sanBuilder = new SubjectAlternativeNameBuilder(); + sanBuilder.AddDnsName(discoveredFqdn); + certRequest.CertificateExtensions.Add(sanBuilder.Build()); using var cert = certRequest.CreateSelfSigned(DateTimeOffset.UtcNow, DateTimeOffset.UtcNow.AddDays(1)); var certPem = cert.ExportCertificatePem(); @@ -1276,6 +1183,227 @@ private static async Task DiscoverFqdnAndBootstrapTlsAsync( } } + /// + /// Extracts the first Hostname-type address from Gateway JSON status. + /// Returns null if no hostname address is found (e.g., only IP addresses). + /// + private static string? ExtractHostnameFromGatewayJson(string gatewayJson) + { + try + { + using var doc = System.Text.Json.JsonDocument.Parse(gatewayJson); + if (doc.RootElement.TryGetProperty("status", out var status) && + status.TryGetProperty("addresses", out var addresses)) + { + // Prefer Hostname-type addresses over IPAddress + foreach (var addr in addresses.EnumerateArray()) + { + if (addr.TryGetProperty("type", out var type) && + string.Equals(type.GetString(), "Hostname", StringComparison.OrdinalIgnoreCase) && + addr.TryGetProperty("value", out var value)) + { + var hostname = value.GetString()?.Trim(); + if (!string.IsNullOrEmpty(hostname)) + { + return hostname; + } + } + } + + // Fall back to any address that looks like a DNS name (contains a dot, no colons) + foreach (var addr in addresses.EnumerateArray()) + { + if (addr.TryGetProperty("value", out var value)) + { + var addrValue = value.GetString()?.Trim(); + if (!string.IsNullOrEmpty(addrValue) && addrValue.Contains('.') && !addrValue.Contains(':')) + { + return addrValue; + } + } + } + } + } + catch (System.Text.Json.JsonException) + { + // Gateway JSON was malformed + } + + return null; + } + + /// + /// Finds HTTPS listener indices that don't have a hostname set, by parsing the + /// full Gateway JSON from kubectl. + /// + private static async Task> FindHostnamelessHttpsListeners( + string gatewayName, + string @namespace, + KubernetesEnvironmentResource environment, + PipelineStepContext context) + { + var getArgs = $"get gateway {gatewayName} --namespace {@namespace} -o json"; + if (environment.KubeConfigPath is not null) + { + getArgs += $" --kubeconfig \"{environment.KubeConfigPath}\""; + } + + var stdout = new List(); + var (getResult, getDisposable) = ProcessUtil.Run(new ProcessSpec("kubectl") + { + Arguments = getArgs, + ThrowOnNonZeroReturnCode = false, + InheritEnv = true, + OnOutputData = stdout.Add, + OnErrorData = _ => { } + }); + + var indices = new List(); + await using (getDisposable.ConfigureAwait(false)) + { + var result = await getResult.WaitAsync(context.CancellationToken).ConfigureAwait(false); + if (result.ExitCode == 0 && stdout.Count > 0) + { + try + { + using var doc = System.Text.Json.JsonDocument.Parse(string.Join("", stdout)); + if (doc.RootElement.TryGetProperty("spec", out var spec) && + spec.TryGetProperty("listeners", out var listeners)) + { + for (var i = 0; i < listeners.GetArrayLength(); i++) + { + var listener = listeners[i]; + if (listener.TryGetProperty("protocol", out var protocol) && + string.Equals(protocol.GetString(), "HTTPS", StringComparison.OrdinalIgnoreCase) && + !listener.TryGetProperty("hostname", out _)) + { + indices.Add(i); + } + } + } + } + catch (System.Text.Json.JsonException) + { + // Fall back to assuming index 1 (HTTP=0, HTTPS=1) + indices.Add(1); + } + } + else + { + // Fall back to assuming index 1 + indices.Add(1); + } + } + + return indices; + } + + /// + /// Transfers field ownership of the Gateway's patched hostname to Helm using server-side apply + /// with a minimal Gateway manifest, avoiding server-populated fields like status and resourceVersion. + /// + private static async Task TransferGatewayFieldOwnership( + string gatewayName, + string @namespace, + KubernetesEnvironmentResource environment, + PipelineStepContext context) + { + // Build a minimal Gateway JSON with only the fields needed for ownership transfer. + // We read the current Gateway, strip server-populated fields, update the hostname, + // and re-apply with Helm's field manager. + var getArgs = $"get gateway {gatewayName} --namespace {@namespace} -o json"; + if (environment.KubeConfigPath is not null) + { + getArgs += $" --kubeconfig \"{environment.KubeConfigPath}\""; + } + + var stdout = new List(); + var (getResult, getDisposable) = ProcessUtil.Run(new ProcessSpec("kubectl") + { + Arguments = getArgs, + ThrowOnNonZeroReturnCode = false, + InheritEnv = true, + OnOutputData = stdout.Add, + OnErrorData = _ => { } + }); + + await using (getDisposable.ConfigureAwait(false)) + { + var exitResult = await getResult.WaitAsync(context.CancellationToken).ConfigureAwait(false); + if (exitResult.ExitCode != 0 || stdout.Count == 0) + { + context.Logger.LogDebug("Could not read Gateway for field ownership transfer."); + return; + } + } + + try + { + using var doc = System.Text.Json.JsonDocument.Parse(string.Join("", stdout)); + var root = doc.RootElement; + + // Build a minimal manifest: apiVersion, kind, metadata (name + namespace only), spec + var minimal = new System.Text.Json.Nodes.JsonObject + { + ["apiVersion"] = root.GetProperty("apiVersion").GetString(), + ["kind"] = root.GetProperty("kind").GetString(), + ["metadata"] = new System.Text.Json.Nodes.JsonObject + { + ["name"] = gatewayName, + ["namespace"] = @namespace + } + }; + + // Copy the spec as-is (it already has the patched hostname from the previous step) + if (root.TryGetProperty("spec", out var spec)) + { + minimal["spec"] = System.Text.Json.Nodes.JsonNode.Parse(spec.GetRawText()); + } + + var tempDir = Directory.CreateTempSubdirectory(".aspire-gateway-ownership"); + try + { + var manifestPath = Path.Combine(tempDir.FullName, "gateway.json"); + await File.WriteAllTextAsync(manifestPath, minimal.ToJsonString(), context.CancellationToken).ConfigureAwait(false); + + var applyArgs = $"apply --server-side --field-manager=helm --force-conflicts -f \"{manifestPath}\""; + if (environment.KubeConfigPath is not null) + { + applyArgs += $" --kubeconfig \"{environment.KubeConfigPath}\""; + } + + var (applyResult, applyDisposable) = ProcessUtil.Run(new ProcessSpec("kubectl") + { + Arguments = applyArgs, + ThrowOnNonZeroReturnCode = false, + InheritEnv = true, + OnOutputData = line => context.Logger.LogDebug("{Line}", line), + OnErrorData = line => context.Logger.LogDebug("{Line}", line) + }); + + await using (applyDisposable.ConfigureAwait(false)) + { + var applyExitResult = await applyResult.WaitAsync(context.CancellationToken).ConfigureAwait(false); + if (applyExitResult.ExitCode != 0) + { + context.Logger.LogDebug( + "Failed to transfer field ownership to Helm (exit code {ExitCode}). " + + "Subsequent deploys with an explicit hostname may require --force.", + applyExitResult.ExitCode); + } + } + } + finally + { + try { tempDir.Delete(recursive: true); } catch { } + } + } + catch (System.Text.Json.JsonException ex) + { + context.Logger.LogDebug(ex, "Failed to parse Gateway JSON for field ownership transfer."); + } + } + private static async Task BootstrapTlsSecretsAsync( PipelineStepContext context, KubernetesEnvironmentResource environment, @@ -1325,6 +1453,9 @@ private static async Task BootstrapTlsSecretsAsync( using var ecdsa = ECDsa.Create(ECCurve.NamedCurves.nistP256); var request = new CertificateRequest($"CN={hostname}", ecdsa, HashAlgorithmName.SHA256); + var sanBuilder = new SubjectAlternativeNameBuilder(); + sanBuilder.AddDnsName(hostname); + request.CertificateExtensions.Add(sanBuilder.Build()); using var cert = request.CreateSelfSigned(DateTimeOffset.UtcNow, DateTimeOffset.UtcNow.AddDays(1)); var certPem = cert.ExportCertificatePem(); From 070bc04e6643cf5a97ee14c0d9e773389393ace5 Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Wed, 29 Apr 2026 19:50:14 +1000 Subject: [PATCH 06/14] Preserve annotations and labels in Gateway ownership transfer The minimal manifest used for server-side apply field ownership transfer was missing annotations and labels, causing AGC annotations like alb.networking.azure.io/alb-name to be stripped. Now copies annotations and labels from the current Gateway metadata. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../KubernetesEnvironmentResource.cs | 28 +++++++++++++++---- 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/src/Aspire.Hosting.Kubernetes/KubernetesEnvironmentResource.cs b/src/Aspire.Hosting.Kubernetes/KubernetesEnvironmentResource.cs index d4e3887333d..05e9bdd1819 100644 --- a/src/Aspire.Hosting.Kubernetes/KubernetesEnvironmentResource.cs +++ b/src/Aspire.Hosting.Kubernetes/KubernetesEnvironmentResource.cs @@ -1342,16 +1342,32 @@ private static async Task TransferGatewayFieldOwnership( using var doc = System.Text.Json.JsonDocument.Parse(string.Join("", stdout)); var root = doc.RootElement; - // Build a minimal manifest: apiVersion, kind, metadata (name + namespace only), spec + // Build a minimal manifest: apiVersion, kind, metadata (name, namespace, annotations, labels), spec + var metadataNode = new System.Text.Json.Nodes.JsonObject + { + ["name"] = gatewayName, + ["namespace"] = @namespace + }; + + // Preserve annotations and labels from the current Gateway + if (root.TryGetProperty("metadata", out var metadata)) + { + if (metadata.TryGetProperty("annotations", out var annotations)) + { + metadataNode["annotations"] = System.Text.Json.Nodes.JsonNode.Parse(annotations.GetRawText()); + } + + if (metadata.TryGetProperty("labels", out var labels)) + { + metadataNode["labels"] = System.Text.Json.Nodes.JsonNode.Parse(labels.GetRawText()); + } + } + var minimal = new System.Text.Json.Nodes.JsonObject { ["apiVersion"] = root.GetProperty("apiVersion").GetString(), ["kind"] = root.GetProperty("kind").GetString(), - ["metadata"] = new System.Text.Json.Nodes.JsonObject - { - ["name"] = gatewayName, - ["namespace"] = @namespace - } + ["metadata"] = metadataNode }; // Copy the spec as-is (it already has the patched hostname from the previous step) From 7461bfa97b5e69d65ae387e0f1f3f7c6ce1d38b5 Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Wed, 29 Apr 2026 21:00:46 +1000 Subject: [PATCH 07/14] Add E2E test for K8S Gateway TLS deployment with HTTP-01 Tests the full flow: provision AKS with ALB controller, install cert-manager with gatewayHTTPRoute HTTP-01 solver, create a project with AddKubernetesEnvironment + AddGateway + WithTls (no hostname), deploy with aspire deploy, and verify FQDN discovery, certificate issuance, and HTTPS access. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../KubernetesGatewayTlsDeploymentTests.cs | 477 ++++++++++++++++++ 1 file changed, 477 insertions(+) create mode 100644 tests/Aspire.Deployment.EndToEnd.Tests/KubernetesGatewayTlsDeploymentTests.cs diff --git a/tests/Aspire.Deployment.EndToEnd.Tests/KubernetesGatewayTlsDeploymentTests.cs b/tests/Aspire.Deployment.EndToEnd.Tests/KubernetesGatewayTlsDeploymentTests.cs new file mode 100644 index 00000000000..8791cc79338 --- /dev/null +++ b/tests/Aspire.Deployment.EndToEnd.Tests/KubernetesGatewayTlsDeploymentTests.cs @@ -0,0 +1,477 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Aspire.Cli.Tests.Utils; +using Aspire.Deployment.EndToEnd.Tests.Helpers; +using Hex1b.Automation; +using Xunit; + +namespace Aspire.Deployment.EndToEnd.Tests; + +/// +/// End-to-end test for deploying an Aspire application to a pre-created AKS cluster +/// with Kubernetes Gateway API + TLS using AddKubernetesEnvironment and AddGateway. +/// The test creates the AKS cluster with ALB controller and Gateway API support, installs +/// cert-manager with a Let's Encrypt HTTP-01 ClusterIssuer (gatewayHTTPRoute solver), then +/// uses aspire deploy with WithTls() (no hostname). It verifies that: +/// 1. The Gateway gets an FQDN assigned by AGC +/// 2. The FQDN discovery pipeline step patches the hostname onto the HTTPS listener +/// 3. cert-manager issues a real TLS certificate via HTTP-01 +/// 4. The app is accessible via port-forward and over HTTPS +/// +public sealed class KubernetesGatewayTlsDeploymentTests(ITestOutputHelper output) +{ + private static readonly TimeSpan s_testTimeout = TimeSpan.FromMinutes(60); + + [Fact] + public async Task DeployStarterWithGatewayTlsToKubernetes() + { + using var cts = new CancellationTokenSource(s_testTimeout); + using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource( + cts.Token, TestContext.Current.CancellationToken); + var cancellationToken = linkedCts.Token; + + await DeployStarterWithGatewayTlsToKubernetesCore(cancellationToken); + } + + private async Task DeployStarterWithGatewayTlsToKubernetesCore(CancellationToken cancellationToken) + { + var subscriptionId = AzureAuthenticationHelpers.TryGetSubscriptionId(); + if (string.IsNullOrEmpty(subscriptionId)) + { + Assert.Skip("Azure subscription not configured. Set ASPIRE_DEPLOYMENT_TEST_SUBSCRIPTION."); + } + + if (!AzureAuthenticationHelpers.IsAzureAuthAvailable()) + { + if (DeploymentE2ETestHelpers.IsRunningInCI) + { + Assert.Fail("Azure authentication not available in CI. Check OIDC configuration."); + } + else + { + Assert.Skip("Azure authentication not available. Run 'az login' to authenticate."); + } + } + + var workspace = TemporaryWorkspace.Create(output); + var startTime = DateTime.UtcNow; + + var resourceGroupName = DeploymentE2ETestHelpers.GenerateResourceGroupName("k8sgwtls"); + var clusterName = $"aks-{DeploymentE2ETestHelpers.GetRunId()}-{DeploymentE2ETestHelpers.GetRunAttempt()}"; + var acrName = $"acrgw{DeploymentE2ETestHelpers.GetRunId()}{DeploymentE2ETestHelpers.GetRunAttempt()}".ToLowerInvariant(); + acrName = new string(acrName.Where(char.IsLetterOrDigit).Take(50).ToArray()); + if (acrName.Length < 5) + { + acrName = $"acrtest{Guid.NewGuid():N}"[..24]; + } + + var projectName = "K8sGatewayTls"; + var k8sNamespace = "gwtls"; + + output.WriteLine($"Test: {nameof(DeployStarterWithGatewayTlsToKubernetes)}"); + output.WriteLine($"Project Name: {projectName}"); + output.WriteLine($"Resource Group: {resourceGroupName}"); + output.WriteLine($"AKS Cluster: {clusterName}"); + output.WriteLine($"ACR Name: {acrName}"); + output.WriteLine($"K8s Namespace: {k8sNamespace}"); + output.WriteLine($"Subscription: {subscriptionId[..8]}..."); + output.WriteLine($"Workspace: {workspace.WorkspaceRoot.FullName}"); + + try + { + using var terminal = DeploymentE2ETestHelpers.CreateTestTerminal(); + var pendingRun = terminal.RunAsync(cancellationToken); + + var counter = new SequenceCounter(); + var auto = new Hex1bTerminalAutomator(terminal, defaultTimeout: TimeSpan.FromSeconds(500)); + + // ===== PHASE 1: Provision AKS with ALB + cert-manager ===== + + output.WriteLine("Step 1: Preparing environment..."); + await auto.PrepareEnvironmentAsync(workspace, counter); + + // Register resource providers for AGC + Gateway API + output.WriteLine("Step 2: Registering resource providers..."); + await auto.TypeAsync( + "az provider register --namespace Microsoft.ContainerService --wait && " + + "az provider register --namespace Microsoft.ContainerRegistry --wait && " + + "az provider register --namespace Microsoft.Network --wait && " + + "az provider register --namespace Microsoft.NetworkFunction --wait && " + + "az provider register --namespace Microsoft.ServiceNetworking --wait"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromMinutes(5)); + + await auto.TypeAsync( + "az feature register --namespace Microsoft.ContainerService --name ManagedGatewayAPIPreview 2>/dev/null || true && " + + "az feature register --namespace Microsoft.ContainerService --name ApplicationLoadBalancerPreview 2>/dev/null || true && " + + "az provider register --namespace Microsoft.ContainerService"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromMinutes(2)); + + await auto.TypeAsync("az extension add --name alb --yes 2>/dev/null || true && az extension add --name aks-preview --yes 2>/dev/null || true"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(60)); + + // Create resource group + output.WriteLine("Step 3: Creating resource group..."); + await auto.TypeAsync($"az group create --name {resourceGroupName} --location westus3 --output table"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(60)); + + // Create ACR + output.WriteLine("Step 4: Creating ACR..."); + await auto.TypeAsync($"az acr create --resource-group {resourceGroupName} --name {acrName} --sku Basic --output table"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromMinutes(3)); + + // Login to ACR early (OIDC token expires during AKS creation) + await auto.TypeAsync($"az acr login --name {acrName}"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(60)); + + // Create AKS with OIDC + workload identity (required for ALB controller) + output.WriteLine("Step 5: Creating AKS cluster (10-15 minutes)..."); + await auto.TypeAsync( + $"az aks create " + + $"--resource-group {resourceGroupName} " + + $"--name {clusterName} " + + $"--node-count 1 " + + $"--node-vm-size Standard_D2s_v3 " + + $"--generate-ssh-keys " + + $"--attach-acr {acrName} " + + $"--enable-managed-identity " + + $"--enable-oidc-issuer " + + $"--enable-workload-identity " + + $"--output table"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromMinutes(20)); + + // Enable ALB controller addon + output.WriteLine("Step 6: Enabling ALB controller..."); + await auto.TypeAsync($"az aks update --resource-group {resourceGroupName} --name {clusterName} --enable-alb"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromMinutes(10)); + + // Get credentials + await auto.TypeAsync($"az aks get-credentials --resource-group {resourceGroupName} --name {clusterName} --overwrite-existing"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(30)); + + // Create ALB subnet + ApplicationLoadBalancer CRD + output.WriteLine("Step 7: Creating ALB subnet and ApplicationLoadBalancer..."); + await auto.TypeAsync( + $"MC_RG=$(az aks show -g {resourceGroupName} -n {clusterName} --query nodeResourceGroup -o tsv) && " + + "VNET_NAME=$(az network vnet list -g $MC_RG --query '[0].name' -o tsv) && " + + "az network vnet subnet create -g $MC_RG --vnet-name $VNET_NAME --name subnet-alb " + + "--address-prefix 10.237.0.0/24 --delegations Microsoft.ServiceNetworking/trafficControllers && " + + "SUBNET_ID=$(az network vnet subnet show -g $MC_RG --vnet-name $VNET_NAME --name subnet-alb --query id -o tsv) && " + + "echo \"Subnet: $SUBNET_ID\""); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromMinutes(3)); + + await auto.TypeAsync( + "cat </dev/null); " + + "[ \"$STATUS\" = \"True\" ] && echo 'ALB Ready' && break; sleep 5; done"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromMinutes(6)); + + // Install cert-manager with Gateway API support + output.WriteLine("Step 8: Installing cert-manager..."); + await auto.TypeAsync( + "helm upgrade --install cert-manager oci://quay.io/jetstack/charts/cert-manager " + + "--namespace cert-manager --create-namespace " + + "--set crds.enabled=true --set config.enableGatewayAPI=true --wait"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromMinutes(5)); + + // Create HTTP-01 ClusterIssuer + output.WriteLine("Step 9: Creating HTTP-01 ClusterIssuer..."); + await auto.TypeAsync( + "cat < + { + helm.WithNamespace(builder.AddParameter("namespace")); + helm.WithChartVersion(builder.AddParameter("chartversion")); + }); + +var gateway = k8s.AddGateway("ingress") + .WithGatewayClass("azure-alb-external") + .WithGatewayAnnotation("alb.networking.azure.io/alb-name", "alb-aspire") + .WithGatewayAnnotation("alb.networking.azure.io/alb-namespace", "default") + .WithGatewayAnnotation("cert-manager.io/cluster-issuer", "letsencrypt-http01") + .WithRoute("/", webfrontend.GetEndpoint("http")) + .WithTls(); + +builder.Build().Run(); +"""; + + content = content.Replace(buildRunPattern, replacement); + + if (!content.Contains("#pragma warning disable ASPIREPIPELINES001")) + { + content = "#pragma warning disable ASPIREPIPELINES001\n" + content; + } + + File.WriteAllText(appHostFilePath, content); + output.WriteLine("Modified AppHost.cs with AddKubernetesEnvironment + AddGateway + WithTls"); + + // Navigate to AppHost dir + await auto.TypeAsync($"cd {projectName}.AppHost"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter); + + // ===== PHASE 3: Deploy with aspire deploy ===== + + // Refresh ACR login + output.WriteLine("Step 14: Refreshing ACR login..."); + await auto.TypeAsync($"az acr login --name {acrName}"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(60)); + + // Set parameters as environment variables so aspire deploy doesn't prompt + output.WriteLine("Step 15: Setting deployment parameters..."); + await auto.TypeAsync( + $"export Parameters__registryendpoint={acrName}.azurecr.io && " + + $"export Parameters__namespace={k8sNamespace} && " + + "export Parameters__chartversion=0.1.0"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter); + + // Deploy using aspire deploy + output.WriteLine("Step 16: Running aspire deploy..."); + await auto.TypeAsync("aspire deploy"); + await auto.EnterAsync(); + await auto.WaitForPipelineSuccessAsync(timeout: TimeSpan.FromMinutes(15)); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromMinutes(2)); + + // ===== PHASE 4: Verify Gateway TLS ===== + + // Wait for pods + output.WriteLine("Step 17: Waiting for pods..."); + await auto.TypeAsync($"kubectl wait --for=condition=Ready pod --all -n {k8sNamespace} --timeout=300s"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromMinutes(6)); + + await auto.TypeAsync($"kubectl get pods -n {k8sNamespace}"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter); + + // Check Gateway has address + output.WriteLine("Step 18: Checking Gateway address..."); + await auto.TypeAsync( + $"for i in $(seq 1 30); do " + + $"FQDN=$(kubectl get gateway ingress -n {k8sNamespace} -o jsonpath='{{.status.addresses[0].value}}' 2>/dev/null); " + + "[ -n \"$FQDN\" ] && echo \"Gateway FQDN: $FQDN\" && break; sleep 5; done"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromMinutes(3)); + + // Check HTTPS listener has hostname (patched by FQDN discovery step) + output.WriteLine("Step 19: Checking HTTPS listener hostname..."); + await auto.TypeAsync( + $"kubectl get gateway ingress -n {k8sNamespace} " + + "-o jsonpath='{range .spec.listeners[*]}{.name} {.protocol} {.hostname}{\"\\n\"}{end}'"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(30)); + + // Wait for certificate + output.WriteLine("Step 20: Waiting for TLS certificate (up to 10 minutes)..."); + await auto.TypeAsync( + $"for i in $(seq 1 60); do " + + $"READY=$(kubectl get certificate -n {k8sNamespace} -o jsonpath='{{.items[0].status.conditions[?(@.type==\"Ready\")].status}}' 2>/dev/null); " + + "[ \"$READY\" = \"True\" ] && echo 'Certificate Ready!' && break; " + + "echo \"Attempt $i: waiting...\"; sleep 10; done && " + + $"kubectl get certificate -n {k8sNamespace}"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromMinutes(12)); + + // Test HTTPS access + output.WriteLine("Step 21: Testing HTTPS access..."); + await auto.TypeAsync( + $"FQDN=$(kubectl get gateway ingress -n {k8sNamespace} -o jsonpath='{{.status.addresses[0].value}}') && " + + "echo \"Testing: https://$FQDN\" && " + + "OK=0; for i in $(seq 1 10); do sleep 5; " + + "S=$(curl -so /dev/null -w '%{http_code}' -m 10 https://$FQDN/ 2>/dev/null); " + + "[ \"$S\" = \"200\" ] && echo \"HTTPS $S OK\" && OK=1 && break; " + + "echo \"Attempt $i: $S\"; done; " + + "[ \"$OK\" = \"1\" ] || echo 'WARN: HTTPS not 200 yet (cert may still be provisioning)'"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromMinutes(2)); + + // Verify via port-forward (confirms app is running regardless of external TLS) + output.WriteLine("Step 22: Verifying app via port-forward..."); + await auto.TypeAsync($"kubectl port-forward svc/webfrontend-service 18081:8080 -n {k8sNamespace} &"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); + + await auto.TypeAsync("sleep 3"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); + + await auto.TypeAsync( + "OK=0; for i in $(seq 1 10); do sleep 3 && " + + "curl -sf http://localhost:18081/ -o /dev/null -w '%{http_code}' && " + + "echo ' OK' && OK=1 && break; done; " + + "[ \"$OK\" = \"1\" ] || { echo 'FAIL: webfrontend unreachable'; exit 1; }"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(60)); + + await auto.TypeAsync("kill %1 2>/dev/null; true"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); + + // ===== PHASE 5: Cleanup ===== + + output.WriteLine("Step 23: Destroying deployment..."); + await auto.AspireDestroyAsync(counter); + + await auto.TypeAsync("exit"); + await auto.EnterAsync(); + + await pendingRun; + + var duration = DateTime.UtcNow - startTime; + output.WriteLine($"Gateway TLS deployment completed in {duration}"); + + DeploymentReporter.ReportDeploymentSuccess( + nameof(DeployStarterWithGatewayTlsToKubernetes), + resourceGroupName, + new Dictionary + { + ["cluster"] = clusterName, + ["acr"] = acrName, + ["project"] = projectName + }, + duration); + + output.WriteLine("✅ Test passed - Aspire app deployed with Gateway API TLS via HTTP-01!"); + } + catch (Exception ex) + { + var duration = DateTime.UtcNow - startTime; + output.WriteLine($"❌ Test failed after {duration}: {ex.Message}"); + + DeploymentReporter.ReportDeploymentFailure( + nameof(DeployStarterWithGatewayTlsToKubernetes), + resourceGroupName, + ex.Message, + ex.StackTrace); + + throw; + } + finally + { + output.WriteLine($"Cleaning up resource group: {resourceGroupName}"); + await CleanupResourceGroupAsync(resourceGroupName); + } + } + + private async Task CleanupResourceGroupAsync(string resourceGroupName) + { + try + { + var process = new System.Diagnostics.Process + { + StartInfo = new System.Diagnostics.ProcessStartInfo + { + FileName = "az", + Arguments = $"group delete --name {resourceGroupName} --yes --no-wait", + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false + } + }; + + process.Start(); + await process.WaitForExitAsync(); + + if (process.ExitCode == 0) + { + output.WriteLine($"Resource group deletion initiated: {resourceGroupName}"); + DeploymentReporter.ReportCleanupStatus(resourceGroupName, success: true, "Deletion initiated"); + } + else + { + var error = await process.StandardError.ReadToEndAsync(); + output.WriteLine($"Resource group deletion may have failed (exit code {process.ExitCode}): {error}"); + DeploymentReporter.ReportCleanupStatus(resourceGroupName, success: false, $"Exit code {process.ExitCode}: {error}"); + } + } + catch (Exception ex) + { + output.WriteLine($"Failed to cleanup resource group: {ex.Message}"); + DeploymentReporter.ReportCleanupStatus(resourceGroupName, success: false, ex.Message); + } + } +} From 362b24aad2271c8e63d34e564966fe2f844cf5e0 Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Wed, 29 Apr 2026 21:46:30 +1000 Subject: [PATCH 08/14] Use --enable-alb in az aks create and AMD VM SKU Consolidate ALB enablement into the az aks create command instead of a separate az aks update step. Use Standard_D2as_v5 (AMD) for quota compatibility with E2E test subscription. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../KubernetesGatewayTlsDeploymentTests.cs | 47 +++++++++---------- 1 file changed, 21 insertions(+), 26 deletions(-) diff --git a/tests/Aspire.Deployment.EndToEnd.Tests/KubernetesGatewayTlsDeploymentTests.cs b/tests/Aspire.Deployment.EndToEnd.Tests/KubernetesGatewayTlsDeploymentTests.cs index 8791cc79338..60232fa71e1 100644 --- a/tests/Aspire.Deployment.EndToEnd.Tests/KubernetesGatewayTlsDeploymentTests.cs +++ b/tests/Aspire.Deployment.EndToEnd.Tests/KubernetesGatewayTlsDeploymentTests.cs @@ -130,36 +130,31 @@ await auto.TypeAsync( await auto.EnterAsync(); await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(60)); - // Create AKS with OIDC + workload identity (required for ALB controller) - output.WriteLine("Step 5: Creating AKS cluster (10-15 minutes)..."); + // Create AKS with OIDC + workload identity + ALB controller in a single command + output.WriteLine("Step 5: Creating AKS cluster with ALB controller (10-15 minutes)..."); await auto.TypeAsync( $"az aks create " + $"--resource-group {resourceGroupName} " + $"--name {clusterName} " + $"--node-count 1 " + - $"--node-vm-size Standard_D2s_v3 " + + $"--node-vm-size Standard_D2as_v5 " + $"--generate-ssh-keys " + $"--attach-acr {acrName} " + $"--enable-managed-identity " + $"--enable-oidc-issuer " + $"--enable-workload-identity " + + $"--enable-alb " + $"--output table"); await auto.EnterAsync(); await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromMinutes(20)); - // Enable ALB controller addon - output.WriteLine("Step 6: Enabling ALB controller..."); - await auto.TypeAsync($"az aks update --resource-group {resourceGroupName} --name {clusterName} --enable-alb"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromMinutes(10)); - // Get credentials await auto.TypeAsync($"az aks get-credentials --resource-group {resourceGroupName} --name {clusterName} --overwrite-existing"); await auto.EnterAsync(); await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(30)); // Create ALB subnet + ApplicationLoadBalancer CRD - output.WriteLine("Step 7: Creating ALB subnet and ApplicationLoadBalancer..."); + output.WriteLine("Step 6: Creating ALB subnet and ApplicationLoadBalancer..."); await auto.TypeAsync( $"MC_RG=$(az aks show -g {resourceGroupName} -n {clusterName} --query nodeResourceGroup -o tsv) && " + "VNET_NAME=$(az network vnet list -g $MC_RG --query '[0].name' -o tsv) && " + @@ -193,7 +188,7 @@ await auto.TypeAsync( await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromMinutes(6)); // Install cert-manager with Gateway API support - output.WriteLine("Step 8: Installing cert-manager..."); + output.WriteLine("Step 7: Installing cert-manager..."); await auto.TypeAsync( "helm upgrade --install cert-manager oci://quay.io/jetstack/charts/cert-manager " + "--namespace cert-manager --create-namespace " + @@ -202,7 +197,7 @@ await auto.TypeAsync( await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromMinutes(5)); // Create HTTP-01 ClusterIssuer - output.WriteLine("Step 9: Creating HTTP-01 ClusterIssuer..."); + output.WriteLine("Step 8: Creating HTTP-01 ClusterIssuer..."); await auto.TypeAsync( "cat </dev/null); " + @@ -338,7 +333,7 @@ await auto.TypeAsync( await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromMinutes(3)); // Check HTTPS listener has hostname (patched by FQDN discovery step) - output.WriteLine("Step 19: Checking HTTPS listener hostname..."); + output.WriteLine("Step 9: Checking HTTPS listener hostname..."); await auto.TypeAsync( $"kubectl get gateway ingress -n {k8sNamespace} " + "-o jsonpath='{range .spec.listeners[*]}{.name} {.protocol} {.hostname}{\"\\n\"}{end}'"); @@ -346,7 +341,7 @@ await auto.TypeAsync( await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(30)); // Wait for certificate - output.WriteLine("Step 20: Waiting for TLS certificate (up to 10 minutes)..."); + output.WriteLine("Step 9: Waiting for TLS certificate (up to 10 minutes)..."); await auto.TypeAsync( $"for i in $(seq 1 60); do " + $"READY=$(kubectl get certificate -n {k8sNamespace} -o jsonpath='{{.items[0].status.conditions[?(@.type==\"Ready\")].status}}' 2>/dev/null); " + @@ -357,7 +352,7 @@ await auto.TypeAsync( await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromMinutes(12)); // Test HTTPS access - output.WriteLine("Step 21: Testing HTTPS access..."); + output.WriteLine("Step 9: Testing HTTPS access..."); await auto.TypeAsync( $"FQDN=$(kubectl get gateway ingress -n {k8sNamespace} -o jsonpath='{{.status.addresses[0].value}}') && " + "echo \"Testing: https://$FQDN\" && " + @@ -370,7 +365,7 @@ await auto.TypeAsync( await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromMinutes(2)); // Verify via port-forward (confirms app is running regardless of external TLS) - output.WriteLine("Step 22: Verifying app via port-forward..."); + output.WriteLine("Step 9: Verifying app via port-forward..."); await auto.TypeAsync($"kubectl port-forward svc/webfrontend-service 18081:8080 -n {k8sNamespace} &"); await auto.EnterAsync(); await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); @@ -393,7 +388,7 @@ await auto.TypeAsync( // ===== PHASE 5: Cleanup ===== - output.WriteLine("Step 23: Destroying deployment..."); + output.WriteLine("Step 9: Destroying deployment..."); await auto.AspireDestroyAsync(counter); await auto.TypeAsync("exit"); From eeddc430cbed7dc615178152d66a4f4254f66ef1 Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Wed, 29 Apr 2026 22:00:52 +1000 Subject: [PATCH 09/14] Fix E2E test: use --enable-gateway-api --enable-application-load-balancer Per the official AGC quickstart docs, use the correct flags: - --enable-gateway-api: enables Gateway API CRDs - --enable-application-load-balancer: enables ALB controller addon - --network-plugin azure: required Azure CNI - Standard_D2as_v5: AMD VM SKU for quota compatibility - Use the add-on's auto-created aks-appgateway subnet instead of creating one manually Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../KubernetesGatewayTlsDeploymentTests.cs | 36 +++++++++++-------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/tests/Aspire.Deployment.EndToEnd.Tests/KubernetesGatewayTlsDeploymentTests.cs b/tests/Aspire.Deployment.EndToEnd.Tests/KubernetesGatewayTlsDeploymentTests.cs index 60232fa71e1..4ec0fad6d2c 100644 --- a/tests/Aspire.Deployment.EndToEnd.Tests/KubernetesGatewayTlsDeploymentTests.cs +++ b/tests/Aspire.Deployment.EndToEnd.Tests/KubernetesGatewayTlsDeploymentTests.cs @@ -130,20 +130,23 @@ await auto.TypeAsync( await auto.EnterAsync(); await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(60)); - // Create AKS with OIDC + workload identity + ALB controller in a single command - output.WriteLine("Step 5: Creating AKS cluster with ALB controller (10-15 minutes)..."); + // Create AKS with Azure CNI, OIDC, workload identity, Gateway API, and ALB controller + // Per https://learn.microsoft.com/azure/application-gateway/for-containers/quickstart-deploy-application-gateway-for-containers-alb-controller-addon + output.WriteLine("Step 5: Creating AKS cluster with Gateway API + ALB (10-15 minutes)..."); await auto.TypeAsync( $"az aks create " + $"--resource-group {resourceGroupName} " + $"--name {clusterName} " + + $"--location westus3 " + $"--node-count 1 " + $"--node-vm-size Standard_D2as_v5 " + + $"--network-plugin azure " + $"--generate-ssh-keys " + $"--attach-acr {acrName} " + - $"--enable-managed-identity " + $"--enable-oidc-issuer " + $"--enable-workload-identity " + - $"--enable-alb " + + $"--enable-gateway-api " + + $"--enable-application-load-balancer " + $"--output table"); await auto.EnterAsync(); await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromMinutes(20)); @@ -153,17 +156,22 @@ await auto.TypeAsync( await auto.EnterAsync(); await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(30)); - // Create ALB subnet + ApplicationLoadBalancer CRD - output.WriteLine("Step 6: Creating ALB subnet and ApplicationLoadBalancer..."); + // Verify ALB controller is running and GatewayClass exists + output.WriteLine("Step 6: Verifying ALB controller and GatewayClass..."); + await auto.TypeAsync("kubectl get pods -n kube-system | grep alb-controller && kubectl get gatewayclass azure-alb-external"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(60)); + + // Create ApplicationLoadBalancer CRD using the add-on's auto-created subnet + output.WriteLine("Step 7: Creating ApplicationLoadBalancer..."); await auto.TypeAsync( $"MC_RG=$(az aks show -g {resourceGroupName} -n {clusterName} --query nodeResourceGroup -o tsv) && " + - "VNET_NAME=$(az network vnet list -g $MC_RG --query '[0].name' -o tsv) && " + - "az network vnet subnet create -g $MC_RG --vnet-name $VNET_NAME --name subnet-alb " + - "--address-prefix 10.237.0.0/24 --delegations Microsoft.ServiceNetworking/trafficControllers && " + - "SUBNET_ID=$(az network vnet subnet show -g $MC_RG --vnet-name $VNET_NAME --name subnet-alb --query id -o tsv) && " + + "SUBNET_ID=$(az network vnet subnet show -g $MC_RG " + + "--vnet-name $(az network vnet list -g $MC_RG --query '[0].name' -o tsv) " + + "--name aks-appgateway --query id -o tsv) && " + "echo \"Subnet: $SUBNET_ID\""); await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromMinutes(3)); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(60)); await auto.TypeAsync( "cat < Date: Wed, 29 Apr 2026 22:50:18 +1000 Subject: [PATCH 10/14] Wait for ALB controller pods to be Running before checking GatewayClass The ALB controller pods need time to initialize after cluster creation. Poll until pods are Running and GatewayClass azure-alb-external exists, with up to 10 minutes timeout. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../KubernetesGatewayTlsDeploymentTests.cs | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/tests/Aspire.Deployment.EndToEnd.Tests/KubernetesGatewayTlsDeploymentTests.cs b/tests/Aspire.Deployment.EndToEnd.Tests/KubernetesGatewayTlsDeploymentTests.cs index 4ec0fad6d2c..2fd01c3bce7 100644 --- a/tests/Aspire.Deployment.EndToEnd.Tests/KubernetesGatewayTlsDeploymentTests.cs +++ b/tests/Aspire.Deployment.EndToEnd.Tests/KubernetesGatewayTlsDeploymentTests.cs @@ -157,10 +157,19 @@ await auto.TypeAsync( await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(30)); // Verify ALB controller is running and GatewayClass exists - output.WriteLine("Step 6: Verifying ALB controller and GatewayClass..."); - await auto.TypeAsync("kubectl get pods -n kube-system | grep alb-controller && kubectl get gatewayclass azure-alb-external"); - await auto.EnterAsync(); - await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(60)); + // The ALB controller pods may still be initializing after cluster creation, + // so poll until they are running and the GatewayClass is available. + output.WriteLine("Step 6: Waiting for ALB controller and GatewayClass..."); + await auto.TypeAsync( + "for i in $(seq 1 60); do " + + "READY=$(kubectl get pods -n kube-system -l app=alb-controller -o jsonpath='{.items[0].status.phase}' 2>/dev/null); " + + "[ \"$READY\" = \"Running\" ] && kubectl get gatewayclass azure-alb-external >/dev/null 2>&1 && " + + "echo 'ALB controller running and GatewayClass available' && break; " + + "echo \"Attempt $i: ALB controller status=$READY, waiting...\"; sleep 10; done && " + + "kubectl get pods -n kube-system | grep alb-controller && " + + "kubectl get gatewayclass azure-alb-external"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromMinutes(10)); // Create ApplicationLoadBalancer CRD using the add-on's auto-created subnet output.WriteLine("Step 7: Creating ApplicationLoadBalancer..."); From e37854421e1bccf74c28894864fbf35495f4f591 Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Wed, 29 Apr 2026 23:38:57 +1000 Subject: [PATCH 11/14] Fix ClusterIssuer: parentRefs requires name and namespace cert-manager requires parentRefs to include a name. Use 'ingress' to match the Gateway name from AddGateway('ingress'), and include the namespace to match the Helm deploy namespace. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../KubernetesGatewayTlsDeploymentTests.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/Aspire.Deployment.EndToEnd.Tests/KubernetesGatewayTlsDeploymentTests.cs b/tests/Aspire.Deployment.EndToEnd.Tests/KubernetesGatewayTlsDeploymentTests.cs index 2fd01c3bce7..a5c79c3008d 100644 --- a/tests/Aspire.Deployment.EndToEnd.Tests/KubernetesGatewayTlsDeploymentTests.cs +++ b/tests/Aspire.Deployment.EndToEnd.Tests/KubernetesGatewayTlsDeploymentTests.cs @@ -232,6 +232,8 @@ await auto.TypeAsync( " gatewayHTTPRoute:\n" + " parentRefs:\n" + " - kind: Gateway\n" + + $" name: ingress\n" + + $" namespace: {k8sNamespace}\n" + "EOF"); await auto.EnterAsync(); await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(30)); From 98155d8b94cc9a25b688849fe97c3357f3bf6bec Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Thu, 30 Apr 2026 00:33:48 +1000 Subject: [PATCH 12/14] Fix E2E test AppHost: add using directive and pragma suppressions The injected AppHost code needs: - using Aspire.Hosting.Kubernetes for AddGateway extension methods - #pragma warning disable ASPIRECOMPUTE003 for AddContainerRegistry Both prepended to the top of the file alongside ASPIREPIPELINES001. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../KubernetesGatewayTlsDeploymentTests.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/Aspire.Deployment.EndToEnd.Tests/KubernetesGatewayTlsDeploymentTests.cs b/tests/Aspire.Deployment.EndToEnd.Tests/KubernetesGatewayTlsDeploymentTests.cs index a5c79c3008d..35e336e19e7 100644 --- a/tests/Aspire.Deployment.EndToEnd.Tests/KubernetesGatewayTlsDeploymentTests.cs +++ b/tests/Aspire.Deployment.EndToEnd.Tests/KubernetesGatewayTlsDeploymentTests.cs @@ -293,9 +293,11 @@ await auto.TypeAsync( content = content.Replace(buildRunPattern, replacement); + // Add required pragmas and using directive at the top of the file + var topOfFile = "#pragma warning disable ASPIREPIPELINES001\n#pragma warning disable ASPIRECOMPUTE003\nusing Aspire.Hosting.Kubernetes;\n"; if (!content.Contains("#pragma warning disable ASPIREPIPELINES001")) { - content = "#pragma warning disable ASPIREPIPELINES001\n" + content; + content = topOfFile + content; } File.WriteAllText(appHostFilePath, content); From 1f2dfbe1b5bea22c7e7209b01df4a7f9686ef845 Mon Sep 17 00:00:00 2001 From: Mitch Denny Date: Thu, 30 Apr 2026 01:19:18 +1000 Subject: [PATCH 13/14] Fix E2E test: capture webfrontend variable from starter template The starter template generates builder.AddProject('webfrontend') without assigning to a variable. The Gateway route needs a reference to it, so inject 'var webfrontend =' before the AddProject call. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../KubernetesGatewayTlsDeploymentTests.cs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/Aspire.Deployment.EndToEnd.Tests/KubernetesGatewayTlsDeploymentTests.cs b/tests/Aspire.Deployment.EndToEnd.Tests/KubernetesGatewayTlsDeploymentTests.cs index 35e336e19e7..652c5f0f2aa 100644 --- a/tests/Aspire.Deployment.EndToEnd.Tests/KubernetesGatewayTlsDeploymentTests.cs +++ b/tests/Aspire.Deployment.EndToEnd.Tests/KubernetesGatewayTlsDeploymentTests.cs @@ -267,6 +267,17 @@ await auto.TypeAsync( var content = File.ReadAllText(appHostFilePath); + // The starter template doesn't assign webfrontend to a variable. + // Insert "var webfrontend = " before the AddProject("webfrontend") call. + content = content.Replace( + "builder.AddProject Date: Thu, 30 Apr 2026 10:06:04 +1000 Subject: [PATCH 14/14] Refactor FQDN discovery polling to use Polly retry pipeline Replace the manual for-loop retry with a Polly ResiliencePipeline using constant 5s backoff, 60 max attempts, and result-based retry (retries when result is null). Polly.Core is already a transitive dependency via Aspire.Hosting. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../KubernetesEnvironmentResource.cs | 100 +++++++++++------- 1 file changed, 62 insertions(+), 38 deletions(-) diff --git a/src/Aspire.Hosting.Kubernetes/KubernetesEnvironmentResource.cs b/src/Aspire.Hosting.Kubernetes/KubernetesEnvironmentResource.cs index 05e9bdd1819..087948ff7ad 100644 --- a/src/Aspire.Hosting.Kubernetes/KubernetesEnvironmentResource.cs +++ b/src/Aspire.Hosting.Kubernetes/KubernetesEnvironmentResource.cs @@ -14,6 +14,8 @@ using Aspire.Hosting.Utils; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; +using Polly; +using Polly.Retry; namespace Aspire.Hosting.Kubernetes; @@ -977,44 +979,8 @@ private static async Task DiscoverFqdnAndBootstrapTlsAsync( context.Logger.LogInformation( "Waiting for Gateway '{GatewayName}' to be assigned a hostname address...", gatewayName); - string? discoveredFqdn = null; - var maxAttempts = 60; // 5 minutes with 5s intervals - for (var attempt = 0; attempt < maxAttempts; attempt++) - { - var getArgs = $"get gateway {gatewayName} --namespace {@namespace} -o json"; - if (environment.KubeConfigPath is not null) - { - getArgs += $" --kubeconfig \"{environment.KubeConfigPath}\""; - } - - var stdout = new List(); - var (getResult, getDisposable) = ProcessUtil.Run(new ProcessSpec("kubectl") - { - Arguments = getArgs, - ThrowOnNonZeroReturnCode = false, - InheritEnv = true, - OnOutputData = stdout.Add, - OnErrorData = _ => { } - }); - - await using (getDisposable.ConfigureAwait(false)) - { - var result = await getResult.WaitAsync(context.CancellationToken).ConfigureAwait(false); - if (result.ExitCode == 0 && stdout.Count > 0) - { - discoveredFqdn = ExtractHostnameFromGatewayJson(string.Join("", stdout)); - if (discoveredFqdn is not null) - { - break; - } - } - } - - if (attempt < maxAttempts - 1) - { - await Task.Delay(TimeSpan.FromSeconds(5), context.CancellationToken).ConfigureAwait(false); - } - } + var discoveredFqdn = await DiscoverGatewayFqdnAsync( + gatewayName, @namespace, environment, context).ConfigureAwait(false); if (string.IsNullOrEmpty(discoveredFqdn)) { @@ -1183,6 +1149,64 @@ await TransferGatewayFieldOwnership( } } + /// + /// Polls for the Gateway's assigned hostname address using a Polly retry pipeline. + /// Retries up to 60 times with 5-second delays (5 minutes total). + /// + private static async Task DiscoverGatewayFqdnAsync( + string gatewayName, + string @namespace, + KubernetesEnvironmentResource environment, + PipelineStepContext context) + { + var pipeline = new ResiliencePipelineBuilder() + .AddRetry(new RetryStrategyOptions + { + MaxRetryAttempts = 59, + Delay = TimeSpan.FromSeconds(5), + BackoffType = DelayBackoffType.Constant, + ShouldHandle = new PredicateBuilder().HandleResult(r => r is null), + OnRetry = args => + { + context.Logger.LogDebug( + "Gateway '{GatewayName}' address not yet available (attempt {Attempt}).", + gatewayName, args.AttemptNumber + 1); + return default; + } + }) + .Build(); + + return await pipeline.ExecuteAsync(async ct => + { + var getArgs = $"get gateway {gatewayName} --namespace {@namespace} -o json"; + if (environment.KubeConfigPath is not null) + { + getArgs += $" --kubeconfig \"{environment.KubeConfigPath}\""; + } + + var stdout = new List(); + var (getResult, getDisposable) = ProcessUtil.Run(new ProcessSpec("kubectl") + { + Arguments = getArgs, + ThrowOnNonZeroReturnCode = false, + InheritEnv = true, + OnOutputData = stdout.Add, + OnErrorData = _ => { } + }); + + await using (getDisposable.ConfigureAwait(false)) + { + var result = await getResult.WaitAsync(ct).ConfigureAwait(false); + if (result.ExitCode == 0 && stdout.Count > 0) + { + return ExtractHostnameFromGatewayJson(string.Join("", stdout)); + } + } + + return null; + }, context.CancellationToken).ConfigureAwait(false); + } + /// /// Extracts the first Hostname-type address from Gateway JSON status. /// Returns null if no hostname address is found (e.g., only IP addresses).