diff --git a/.gitignore b/.gitignore index 85eb4eac46a..0633d9b0908 100644 --- a/.gitignore +++ b/.gitignore @@ -172,6 +172,8 @@ tests/PolyglotAppHosts/**/Java/**/*.class # Publisher Outputs playground/**/publish/ +playground/**/aspire-output/ +playground/**/aspire-manifest.json # TypeScript AppHost playground/**/dist/ diff --git a/Aspire.slnx b/Aspire.slnx index 30979a3df35..3d69dd9b346 100644 --- a/Aspire.slnx +++ b/Aspire.slnx @@ -174,6 +174,10 @@ + + + + diff --git a/docs/specs/aks-support.md b/docs/specs/aks-support.md index 41a6af54287..7ec95178afa 100644 --- a/docs/specs/aks-support.md +++ b/docs/specs/aks-support.md @@ -589,7 +589,109 @@ var aks = builder.AddAzureKubernetesService("aks") - 🔲 AKS resource does not implement `IAzureContainerRegistry` (ACR outputs not exposed via standard interface) #### Ingress controller -- 🔲 Application Gateway Ingress Controller (AGIC) or other ingress support +- ✅ Azure Application Gateway for Containers (AGC) via `AddLoadBalancer()` + `WithLoadBalancer()` (see below) +- ✅ Cert-manager auto-TLS via `WithTls(issuer)` on Gateway resources (bootstrap secret + post-FQDN cert-manager swap) +- ✅ Helm `WithForceConflicts()` (`--force-conflicts`) for cert-manager / AGC controller SSA field-manager conflicts +- ✅ AGC controller identity auto-granted `Network Contributor` on each ALB subnet +- ✅ `AddHelmChart` / `AddLoadBalancer` skip model registration in run mode (matches `AddIngress` / `AddGateway`) +- 🔲 Cluster-level no-arg `AddLoadBalancer()` (auto VNet + delegated subnet) — left as future work + +### Application Gateway for Containers (AGC) ingress ✅ + +Opt-in, multi-LB ingress wired into Aspire's existing `AddGateway`/`AddIngress` model. The single public entry point is `AddLoadBalancer` on the AKS environment: + +```csharp +var vnet = builder.AddAzureVirtualNetwork("vnet", "10.100.0.0/16"); +var aksSubnet = vnet.AddSubnet("aks", "10.100.0.0/22"); +var alb1Subnet = vnet.AddSubnet("alb1", "10.100.4.0/24"); +var alb2Subnet = vnet.AddSubnet("alb2", "10.100.5.0/24"); + +var aks = builder.AddAzureKubernetesEnvironment("aks").WithSubnet(aksSubnet); + +var lb1 = aks.AddLoadBalancer("public", alb1Subnet); +var lb2 = aks.AddLoadBalancer("admin", alb2Subnet); + +aks.AddGateway("storefront").WithLoadBalancer(lb1); +aks.AddIngress("api").WithLoadBalancer(lb1); +aks.AddGateway("admin-portal").WithLoadBalancer(lb2); +``` + +What `AddLoadBalancer` does: +- Flips internal flags on the AKS environment so its emitted Bicep uses the + `2025-09-02-preview` API version and includes both + `properties.ingressProfile.gatewayAPI.installation: 'Standard'` and + `properties.ingressProfile.applicationLoadBalancer.enabled: true`. The preview API + shape is injected via `AksPreviewIngressProfileInjector`, which uses a small reflection + shim onto `ProvisionableConstruct.DefineProperty` because the underlying + `ManagedClusterIngressProfile` type is `internal` in `Azure.Provisioning.ContainerService` + and cannot be subclassed like the other preview-Bicep injectors do. (These flags are + internal-only — the cluster-level toggles are intentionally not exposed as separate + public extensions to keep the surface small and prevent users from opting into the + preview API by accident.) +- Applies an idempotent `AzureSubnetServiceDelegationAnnotation` for + `Microsoft.ServiceNetworking/trafficControllers` to the supplied subnet (multiple LBs + may share the same subnet). +- Auto-grants `Network Contributor` on each ALB subnet to the AGC controller's + user-assigned identity (the cluster's `applicationLoadBalancer` ingress profile + identity), so the controller can program the subnet without manual role wiring. +- Skips registering the resource in the model in run mode (mirrors `AddIngress` / + `AddGateway` / `AddHelmChart`) so the helm/k8s pipeline isn't pulled in when running + locally. +- Returns an `AzureKubernetesLoadBalancerResource` whose own pipeline step + (`apply-alb-crd-{name}`) waits for the `azure-alb-external` GatewayClass to appear, + then `kubectl apply -f -` an `ApplicationLoadBalancer` CR named `alb-{name}` in the + `default` namespace, pointing at the supplied subnet. + +What `WithLoadBalancer` does on a `KubernetesGatewayResource` / `KubernetesIngressResource`: +- Adds the AGC association annotations + (`alb.networking.azure.io/alb-name: alb-{lb}`, `alb.networking.azure.io/alb-namespace: default`) + to the rendered Helm template. +- Defaults the `gatewayClassName` / `ingressClassName` to `azure-alb-external` if the + user did not set one explicitly. + +Why multi-LB by design: each AGC `ApplicationLoadBalancer` caps at five frontends, so +larger apps need to spread Gateways/Ingresses across multiple LBs. Each +`AzureKubernetesLoadBalancerResource` owns its own pipeline step so apply / wait-ready / +future-destroy lifecycle is per-LB. + +#### TLS via cert-manager (`WithTls`) ✅ + +Gateways can opt into auto-managed HTTPS via cert-manager + Let's Encrypt: + +```csharp +aks.AddHelmChart("cert-manager", "https://charts.jetstack.io", "cert-manager") + .WithValues(new { crds = new { enabled = true } }); + +aks.AddGateway("storefront") + .WithLoadBalancer(lb1) + .WithTls("letsencrypt-prod"); // ClusterIssuer name +``` + +`WithTls(issuer)` does: +- Adds an HTTPS listener (port 443) referencing a TLS secret named `{gateway}-tls`. +- Annotates the Gateway with `cert-manager.io/cluster-issuer: {issuer}` so cert-manager + watches the Gateway and mints a cert into the referenced secret once the AGC frontend + FQDN is discoverable. +- Pre-creates a self-signed bootstrap TLS secret with placeholder hostname + `bootstrap.invalid` **before** waiting for the Gateway FQDN. AGC refuses to program a + Gateway whose HTTPS listener references a non-existent secret, but FQDN discovery + doesn't run until the Gateway is programmed — pre-creating the bootstrap secret breaks + this chicken-and-egg deadlock. Once the FQDN is known, the existing patch logic adds + the discovered hostname so cert-manager can swap in the real cert. + +Force-conflicts for SSA conflicts: helm chart resources support `WithForceConflicts()` which +adds `--force-conflicts` to `helm upgrade`. Required because cert-manager +and AGC controllers patch fields on the same Gateway/secret resources, producing +server-side-apply field-manager conflicts that would otherwise fail subsequent helm +upgrades. + +### AksDemo playground ✅ + +`playground/AksDemo/` exercises the full AKS environment + AGC + Gateway API + cert-manager +TLS path against a live AKS cluster. AppHost uses `AddAzureKubernetesEnvironment` + +`AddLoadBalancer` + multiple `AddGateway` resources sharing one ALB, with +`Standard_D2as_v5` system node pool and VNet at `10.100.0.0/16` (avoids the default AKS +service CIDR `10.0.0.0/16`). #### Managed Prometheus/Grafana - 🔲 Azure Monitor workspace for managed Prometheus @@ -617,4 +719,5 @@ var aks = builder.AddAzureKubernetesService("aks") - 31 AKS unit tests passing (extensions + infrastructure) - 88 K8s base tests passing -- Manual E2E validation against live Azure clusters +- `playground/AksDemo/` validated end-to-end against a live AKS cluster (multi-Gateway, AGC, cert-manager TLS) +- `tests/Aspire.Deployment.EndToEnd.Tests/AksAzureKubernetesEnvironmentGatewayDeploymentTests` — automated deployment test that provisions an AKS environment with AGC + Gateway, deploys an API, and verifies an HTTP 200 from `/weatherforecast`. Passes against live Azure (`westus3`, `Standard_D2as_v5`). diff --git a/playground/AksDemo/AksDemo.ApiService/AksDemo.ApiService.csproj b/playground/AksDemo/AksDemo.ApiService/AksDemo.ApiService.csproj new file mode 100644 index 00000000000..68db7ed9157 --- /dev/null +++ b/playground/AksDemo/AksDemo.ApiService/AksDemo.ApiService.csproj @@ -0,0 +1,13 @@ + + + + $(DefaultTargetFramework) + enable + enable + + + + + + + diff --git a/playground/AksDemo/AksDemo.ApiService/Program.cs b/playground/AksDemo/AksDemo.ApiService/Program.cs new file mode 100644 index 00000000000..0718efd9ed1 --- /dev/null +++ b/playground/AksDemo/AksDemo.ApiService/Program.cs @@ -0,0 +1,52 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +var builder = WebApplication.CreateBuilder(args); + +builder.AddServiceDefaults(); + +var app = builder.Build(); + +app.MapDefaultEndpoints(); + +// Simple endpoints to validate the Gateway -> HTTPRoute -> Service path through AGC. +// / - returns identifying info so it's obvious which pod handled the request +// /hello/{name} - echoes a name so route prefixes can be exercised +// /info - returns environment metadata useful for debugging + +static object BuildIdentity(string surface) => new +{ + service = "AksDemo.ApiService", + surface, + machineName = Environment.MachineName, + podIp = Environment.GetEnvironmentVariable("POD_IP"), + timestampUtc = DateTimeOffset.UtcNow +}; + +app.MapGet("/", () => Results.Ok(BuildIdentity("root"))); + +// The AKS gateways route /api -> storefront-gw and /admin -> admin-gw without +// rewriting the path, so the API needs endpoints under those exact prefixes +// for end-to-end smoke tests through AGC to return 200. +app.MapGet("/api", () => Results.Ok(BuildIdentity("storefront"))); +app.MapGet("/admin", () => Results.Ok(BuildIdentity("admin"))); + +app.MapGet("/hello/{name}", (string name) => Results.Ok(new +{ + message = $"Hello, {name}!", + machineName = Environment.MachineName +})); + +app.MapGet("/info", () => Results.Ok(new +{ + machineName = Environment.MachineName, + osVersion = Environment.OSVersion.ToString(), + processorCount = Environment.ProcessorCount, + dotnetVersion = Environment.Version.ToString(), + aspnetcoreEnv = Environment.GetEnvironmentVariable("ASPNETCORE_ENVIRONMENT"), + podIp = Environment.GetEnvironmentVariable("POD_IP"), + podName = Environment.GetEnvironmentVariable("POD_NAME"), + nodeName = Environment.GetEnvironmentVariable("NODE_NAME") +})); + +app.Run(); diff --git a/playground/AksDemo/AksDemo.ApiService/Properties/launchSettings.json b/playground/AksDemo/AksDemo.ApiService/Properties/launchSettings.json new file mode 100644 index 00000000000..350bdb1180b --- /dev/null +++ b/playground/AksDemo/AksDemo.ApiService/Properties/launchSettings.json @@ -0,0 +1,14 @@ +{ + "$schema": "http://json.schemastore.org/launchsettings.json", + "profiles": { + "http": { + "commandName": "Project", + "dotnetRunMessages": true, + "launchBrowser": false, + "applicationUrl": "http://localhost:5197", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + } + } +} diff --git a/playground/AksDemo/AksDemo.ApiService/appsettings.json b/playground/AksDemo/AksDemo.ApiService/appsettings.json new file mode 100644 index 00000000000..10f68b8c8b4 --- /dev/null +++ b/playground/AksDemo/AksDemo.ApiService/appsettings.json @@ -0,0 +1,9 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Warning" + } + }, + "AllowedHosts": "*" +} diff --git a/playground/AksDemo/AksDemo.AppHost/AksDemo.AppHost.csproj b/playground/AksDemo/AksDemo.AppHost/AksDemo.AppHost.csproj new file mode 100644 index 00000000000..f66d54f6ce1 --- /dev/null +++ b/playground/AksDemo/AksDemo.AppHost/AksDemo.AppHost.csproj @@ -0,0 +1,21 @@ + + + + Exe + $(DefaultTargetFramework) + enable + enable + true + 3f6d6bf5-9d5e-45b1-86b3-629372b14c0e + + + + + + + + + + + + diff --git a/playground/AksDemo/AksDemo.AppHost/AppHost.cs b/playground/AksDemo/AksDemo.AppHost/AppHost.cs new file mode 100644 index 00000000000..7cac801d2c2 --- /dev/null +++ b/playground/AksDemo/AksDemo.AppHost/AppHost.cs @@ -0,0 +1,78 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#pragma warning disable ASPIREAZURE003 // AddSubnet / AzureSubnetResource are evaluation-only + +var builder = DistributedApplication.CreateBuilder(args); + +// VNet layout: +// 10.100.0.0/16 - vnet (chosen to avoid the AKS default service CIDR 10.0.0.0/16) +// 10.100.0.0/22 - aks node pool subnet (1024 IPs - room for pods/nodes) +// 10.100.4.0/24 - public AGC frontend subnet (delegated to ServiceNetworking by AddLoadBalancer) +// 10.100.5.0/24 - admin AGC frontend subnet +// +// AGC requires the ALB frontend subnet to be /24 or larger and to be delegated to +// Microsoft.ServiceNetworking/trafficControllers. AddLoadBalancer applies the delegation +// for us; we just need to make sure the AKS subnet and ALB subnets do not overlap. +var vnet = builder.AddAzureVirtualNetwork("vnet", "10.100.0.0/16"); +var aksSubnet = vnet.AddSubnet("aks-nodes", "10.100.0.0/22"); +var publicSubnet = vnet.AddSubnet("alb-public", "10.100.4.0/24"); +var adminSubnet = vnet.AddSubnet("alb-admin", "10.100.5.0/24"); + +var aks = builder.AddAzureKubernetesEnvironment("aks") + .WithSubnet(aksSubnet) + // Use the same AMD-based SKU as our AKS deployment E2E tests so this + // playground deploys consistently across regions and quotas. + .WithSystemNodePool("Standard_D2as_v5"); + +aks.AddNodePool("workload", "Standard_D2as_v5", minCount: 1, maxCount: 3); + +// Two AGC ApplicationLoadBalancers. Each AGC ALB caps at 5 frontends, so production apps +// often need to spread Gateways/Ingresses across multiple LBs. This playground uses two +// just to exercise the multi-LB code path. +var publicLb = aks.AddLoadBalancer("public", publicSubnet); +var adminLb = aks.AddLoadBalancer("admin", adminSubnet); + +var api = builder.AddProject("api") + .WithExternalHttpEndpoints(); + +// Public gateway: serves /api -> the api service, attached to the public AGC ALB. +// WithLoadBalancer attaches the alb.networking.azure.io association annotations and +// defaults the gatewayClassName to "azure-alb-external". +// +// WithTls() (no hostname) creates an HTTPS listener that gets its hostname patched in +// by Aspire's tls-fqdn-discovery pipeline step once AGC assigns the gateway its +// .fz.alb.azure.com FQDN. The cert-manager.io/cluster-issuer annotation +// then triggers cert-manager to issue a real Let's Encrypt cert via HTTP-01 against +// that FQDN. A `letsencrypt-prod` ClusterIssuer needs to exist in the cluster. +aks.AddGateway("storefront-gw") + .WithLoadBalancer(publicLb) + .WithRoute("/api", api.GetEndpoint("http")) + .WithTls() + .WithGatewayAnnotation("cert-manager.io/cluster-issuer", "letsencrypt-prod"); + +// Admin gateway: serves the same backend but on a separate AGC ALB so a different set of +// network policies, frontends, or DNS names can hang off it. +aks.AddGateway("admin-gw") + .WithLoadBalancer(adminLb) + .WithRoute("/admin", api.GetEndpoint("http")); + +// cert-manager installed via Helm so we can issue Let's Encrypt certificates for the AGC +// gateways via the HTTP-01 challenge. Gateway API support is enabled so cert-manager will +// watch Gateway listeners for TLS configuration and auto-issue Certificates. +// +// WithForceConflicts is needed because AKS clusters with the Azure Policy add-on (or +// Deployment Safeguards) install an `admissionsenforcer` field manager that mutates the +// cert-manager ValidatingWebhookConfiguration after the first install. Helm's SSA then +// fails the next upgrade with a conflict on .webhooks[*].namespaceSelector. +// WithForceConflicts adds --force-conflicts which tells SSA to take over the conflicting +// field non-destructively (no resources recreated). +aks.AddHelmChart("cert-manager", "oci://quay.io/jetstack/charts/cert-manager", "v1.18.2") + .WithHelmValue("crds.enabled", "true") + .WithHelmValue("config.apiVersion", "controller.config.cert-manager.io/v1alpha1") + .WithHelmValue("config.kind", "ControllerConfiguration") + .WithHelmValue("config.enableGatewayAPI", "true") + .WithForceConflicts() + .WithDestroy(); + +builder.Build().Run(); diff --git a/playground/AksDemo/AksDemo.AppHost/Properties/launchSettings.json b/playground/AksDemo/AksDemo.AppHost/Properties/launchSettings.json new file mode 100644 index 00000000000..c981b83d13c --- /dev/null +++ b/playground/AksDemo/AksDemo.AppHost/Properties/launchSettings.json @@ -0,0 +1,44 @@ +{ + "$schema": "http://json.schemastore.org/launchsettings.json", + "profiles": { + "https": { + "commandName": "Project", + "dotnetRunMessages": true, + "launchBrowser": true, + "applicationUrl": "https://localhost:16140;http://localhost:16141", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development", + "DOTNET_ENVIRONMENT": "Development", + "ASPIRE_DASHBOARD_OTLP_ENDPOINT_URL": "https://localhost:17060", + "ASPIRE_RESOURCE_SERVICE_ENDPOINT_URL": "https://localhost:17060", + "ASPIRE_SHOW_DASHBOARD_RESOURCES": "true" + } + }, + "http": { + "commandName": "Project", + "dotnetRunMessages": true, + "launchBrowser": true, + "applicationUrl": "http://localhost:16141", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development", + "DOTNET_ENVIRONMENT": "Development", + "ASPIRE_DASHBOARD_OTLP_ENDPOINT_URL": "http://localhost:17061", + "ASPIRE_RESOURCE_SERVICE_ENDPOINT_URL": "http://localhost:17061", + "ASPIRE_SHOW_DASHBOARD_RESOURCES": "true", + "ASPIRE_ALLOW_UNSECURED_TRANSPORT": "true" + } + }, + "generate-manifest": { + "commandName": "Project", + "launchBrowser": true, + "dotnetRunMessages": true, + "commandLineArgs": "--publisher manifest --output-path aspire-manifest.json", + "applicationUrl": "http://localhost:16141", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development", + "DOTNET_ENVIRONMENT": "Development", + "ASPIRE_DASHBOARD_OTLP_ENDPOINT_URL": "http://localhost:17061" + } + } + } +} diff --git a/playground/AksDemo/AksDemo.AppHost/appsettings.json b/playground/AksDemo/AksDemo.AppHost/appsettings.json new file mode 100644 index 00000000000..31c092aa450 --- /dev/null +++ b/playground/AksDemo/AksDemo.AppHost/appsettings.json @@ -0,0 +1,9 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Warning", + "Aspire.Hosting.Dcp": "Warning" + } + } +} diff --git a/playground/AksDemo/aspire.config.json b/playground/AksDemo/aspire.config.json new file mode 100644 index 00000000000..5bb6372b56f --- /dev/null +++ b/playground/AksDemo/aspire.config.json @@ -0,0 +1,5 @@ +{ + "appHost": { + "path": "AksDemo.AppHost/AksDemo.AppHost.csproj" + } +} diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AksPreviewIngressProfileInjector.cs b/src/Aspire.Hosting.Azure.Kubernetes/AksPreviewIngressProfileInjector.cs new file mode 100644 index 00000000000..cd0fc8361b0 --- /dev/null +++ b/src/Aspire.Hosting.Azure.Kubernetes/AksPreviewIngressProfileInjector.cs @@ -0,0 +1,183 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Reflection; +using Azure.Provisioning; +using Azure.Provisioning.ContainerService; +using Azure.Provisioning.Primitives; + +namespace Aspire.Hosting.Azure.Kubernetes; + +/// +/// Injects the AKS preview-only properties.ingressProfile.gatewayAPI.installation +/// and properties.ingressProfile.applicationLoadBalancer.enabled Bicep properties +/// onto a . These properties only exist on the +/// 2025-08-02-preview (gatewayAPI) and 2025-09-02-preview +/// (applicationLoadBalancer) AKS API versions; the latest stable 2026-01-01 and +/// Azure.Provisioning.ContainerService 1.0.0-beta.6 expose neither. +/// +/// +/// +/// Reflection is unavoidable here. The Provisioning emitter merges sibling property +/// declarations only when they are registered on the same +/// instance. ManagedClusterIngressProfile (the typed parent of gatewayAPI and +/// applicationLoadBalancer) is internal in +/// Azure.Provisioning.ContainerService 1.0.0-beta.6, so we cannot subclass it to add +/// the missing properties through the normal public extension pattern (compare +/// ContainerAppEnvironmentDotnetComponentResource in +/// Aspire.Hosting.Azure.AppContainers, CosmosDBSqlRoleAssignment_Derived in +/// Aspire.Hosting.Azure.CosmosDB, and +/// PublicHostingCognitiveServicesCapabilityHostProperties in +/// Aspire.Hosting.Foundry, all of which extend public typed parents). +/// +/// +/// Two reflection-free alternatives were attempted and empirically ruled out: +/// +/// +/// +/// +/// Subclassing and calling +/// DefineProperty<T> with a deep path such as +/// ["properties", "ingressProfile", "gatewayAPI", "installation"]. The deeper +/// declaration shadows the typed Properties declaration on the base, so the +/// emitted Bicep loses dnsPrefix, agentPoolProfiles, +/// oidcIssuerProfile and securityProfile. +/// +/// +/// +/// +/// Same subclass plus DefineModelProperty<T>(..., ["properties", "ingressProfile"], new T()) +/// grafting a public that internally registers +/// ["gatewayAPI", "installation"] and ["applicationLoadBalancer", "enabled"]. +/// The grafted model emits its own properties correctly, but still shadows the typed +/// Properties declaration on the base for the same reason as (1). +/// +/// +/// +/// +/// The reflection path works because DefineProperty<T> called on the inner +/// internal ManagedClusterIngressProfile instance produces a +/// whose path is rooted at that instance — it merges with the +/// sibling typed webAppRouting declaration on the same construct rather than +/// shadowing the cluster-level Properties. +/// +/// +/// To make the inner IngressProfile instance exist (it is lazily created when the +/// public setter is +/// invoked), we assign an empty . +/// An empty inner webAppRouting object is filtered out by the emitter, so the +/// rendered Bicep does not gain a stray webAppRouting: {} block. +/// +/// +/// The proper long-term fix is for Azure.Provisioning.ContainerService to expose +/// ManagedClusterIngressProfile publicly (or to add typed GatewayApi and +/// ApplicationLoadBalancer properties on it). When that ships, this class can be +/// replaced with the standard public-subclass pattern and deleted. +/// +/// +/// Tracked by (Aspire) and +/// (upstream). +/// +/// +// TODO: https://github.com/microsoft/aspire/issues/17060 - delete this class once +// Azure.Provisioning.ContainerService exposes ManagedClusterIngressProfile publicly +// with typed GatewayApi and ApplicationLoadBalancer properties. +internal static class AksPreviewIngressProfileInjector +{ + private const string IngressProfileTypeFullName = "Azure.Provisioning.ContainerService.ManagedClusterIngressProfile"; + + private static readonly Lazy s_defineProperty = new(() => + { + // protected BicepValue DefineProperty(string propertyName, string[] bicepPath, bool isOutput = false, bool isRequired = false, bool isSecure = false, BicepValue? defaultValue = null, string? format = null) + return typeof(ProvisionableConstruct).GetMethod( + "DefineProperty", + BindingFlags.NonPublic | BindingFlags.Instance) + ?? throw new InvalidOperationException("ProvisionableConstruct.DefineProperty not found via reflection. The Azure.Provisioning surface may have changed; review AksPreviewIngressProfileInjector."); + }); + + /// + /// Injects the requested preview-only ingressProfile entries onto . + /// Caller is responsible for setting an appropriate preview ResourceVersion on + /// the cluster (e.g. 2025-09-02-preview) before any properties are compiled. + /// + public static void Inject(ContainerServiceManagedCluster aks, bool gatewayApi, bool applicationLoadBalancer) + { + if (!gatewayApi && !applicationLoadBalancer) + { + return; + } + + // Bootstrap the lazily-created internal IngressProfile by assigning an empty + // WebAppRouting object via the public setter. An empty WebAppRouting object is + // filtered out at emission time, so this does not introduce a stray webAppRouting + // entry into the rendered Bicep. + aks.IngressWebAppRouting = new ManagedClusterIngressProfileWebAppRouting(); + + var ingressProfile = GetIngressProfileInstance(aks); + var defineProperty = s_defineProperty.Value; + + if (gatewayApi) + { + // properties.ingressProfile.gatewayAPI.installation = 'Standard' + // The only AKS-managed installation value is "Standard"; it installs the + // upstream Gateway API CRDs and the AKS-managed Gateway controller. + var installation = (BicepValue)defineProperty + .MakeGenericMethod(typeof(string)) + .Invoke(ingressProfile, [ + "GatewayAPIInstallation", + new[] { "gatewayAPI", "installation" }, + /* isOutput */ false, + /* isRequired */ false, + /* isSecure */ false, + /* defaultValue */ null, + /* format */ null, + ])!; + installation.Assign("Standard"); + } + + if (applicationLoadBalancer) + { + // properties.ingressProfile.applicationLoadBalancer.enabled = true + // Enables the AKS-managed AGC ALB controller add-on (which installs the + // azure-alb-external GatewayClass and watches for ApplicationLoadBalancer CRs). + var enabled = (BicepValue)defineProperty + .MakeGenericMethod(typeof(bool)) + .Invoke(ingressProfile, [ + "ApplicationLoadBalancerEnabled", + new[] { "applicationLoadBalancer", "enabled" }, + /* isOutput */ false, + /* isRequired */ false, + /* isSecure */ false, + /* defaultValue */ null, + /* format */ null, + ])!; + enabled.Assign(true); + } + } + + private static object GetIngressProfileInstance(ContainerServiceManagedCluster aks) + { + // ContainerServiceManagedCluster.Properties is internal and exposes the + // ManagedClusterProperties complex object that owns IngressProfile (also internal). + var clusterPropsProp = typeof(ContainerServiceManagedCluster).GetProperty( + "Properties", + BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance) + ?? throw new InvalidOperationException("ContainerServiceManagedCluster.Properties not found via reflection. The Azure.Provisioning surface may have changed; review AksPreviewIngressProfileInjector."); + var clusterProps = clusterPropsProp.GetValue(aks) + ?? throw new InvalidOperationException("ContainerServiceManagedCluster.Properties returned null."); + + var ipProp = clusterProps.GetType().GetProperty( + "IngressProfile", + BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance) + ?? throw new InvalidOperationException("ManagedClusterProperties.IngressProfile not found via reflection. The Azure.Provisioning surface may have changed; review AksPreviewIngressProfileInjector."); + var ipInstance = ipProp.GetValue(clusterProps) + ?? throw new InvalidOperationException("ManagedClusterProperties.IngressProfile was null even after assigning IngressWebAppRouting; the Azure.Provisioning lazy-initialization behavior may have changed."); + + if (ipInstance.GetType().FullName != IngressProfileTypeFullName) + { + throw new InvalidOperationException($"Expected IngressProfile to be a {IngressProfileTypeFullName} but found {ipInstance.GetType().FullName}."); + } + + return ipInstance; + } +} diff --git a/src/Aspire.Hosting.Azure.Kubernetes/Aspire.Hosting.Azure.Kubernetes.csproj b/src/Aspire.Hosting.Azure.Kubernetes/Aspire.Hosting.Azure.Kubernetes.csproj index 75dddab3ab7..d58a1c8436a 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/Aspire.Hosting.Azure.Kubernetes.csproj +++ b/src/Aspire.Hosting.Azure.Kubernetes/Aspire.Hosting.Azure.Kubernetes.csproj @@ -24,6 +24,7 @@ + diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs index e722d41626a..fd5498b4696 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs +++ b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentExtensions.cs @@ -16,6 +16,7 @@ using Azure.Provisioning.ContainerRegistry; using Azure.Provisioning.ContainerService; using Azure.Provisioning.Expressions; +using Azure.Provisioning.Network; using Azure.Provisioning.Resources; using Azure.Provisioning.Roles; using Microsoft.Extensions.DependencyInjection; @@ -349,6 +350,134 @@ public static IResourceBuilder WithContainer return builder; } + /// + /// Adds an Azure Application Gateway for Containers (AGC) ApplicationLoadBalancer + /// to this AKS environment, bound to the supplied delegated subnet. Returns a resource + /// builder that can be passed to gateway.WithLoadBalancer(lb) / + /// ingress.WithLoadBalancer(lb) to route traffic through this load balancer. + /// + /// The AKS environment resource builder. + /// The name of the load balancer resource. Used to derive the in-cluster + /// ApplicationLoadBalancer name (alb-{name}) referenced by gateway/ingress annotations. + /// A subnet that will be associated with the AGC ALB. The subnet is + /// automatically delegated to Microsoft.ServiceNetworking/trafficControllers; this is + /// required by AGC and is idempotent across multiple calls + /// against the same subnet. + /// A reference to the . + /// + /// + /// Each AGC ApplicationLoadBalancer caps at 5 frontends, so applications that need + /// more should call AddLoadBalancer multiple times (each call may use the same or a + /// different subnet) and pin gateways/ingresses to specific load balancers via + /// . + /// + /// + /// Calling this method opts the AKS cluster into the managed Gateway API installation + /// (ingressProfile.gatewayAPI.installation = 'Standard') and the AGC ALB controller + /// add-on (ingressProfile.applicationLoadBalancer.enabled = true). Both properties + /// only exist in preview AKS Bicep API versions (oldest covering both: 2025-09-02-preview), + /// so this implicitly bumps the cluster's emitted API version. Subscriptions/regions where + /// the AKS preview features Microsoft.ContainerService/AKSGatewayAPIPreview and + /// Microsoft.ContainerService/AKSAppGatewayContainersPreview are not registered will + /// see deployment failures. + /// + /// + /// After provisioning, a per-LB pipeline step (apply-alb-crd-{name}) waits for the + /// azure-alb-external GatewayClass to appear in the cluster and then + /// kubectl applys the ApplicationLoadBalancer custom resource pointing at the + /// supplied subnet. + /// + /// + /// + /// + /// var vnet = builder.AddAzureVirtualNetwork("vnet", "10.0.0.0/16"); + /// var aksSubnet = vnet.AddSubnet("aks", "10.0.0.0/22"); + /// var albSubnet = vnet.AddSubnet("alb", "10.0.4.0/24"); + /// + /// var aks = builder.AddAzureKubernetesEnvironment("aks").WithSubnet(aksSubnet); + /// var lb = aks.AddLoadBalancer("lb", albSubnet); + /// + /// aks.AddGateway("public").WithLoadBalancer(lb); + /// + /// + [AspireExport(Description = "Adds an Azure Application Gateway for Containers ApplicationLoadBalancer to the AKS environment")] + public static IResourceBuilder AddLoadBalancer( + this IResourceBuilder builder, + [ResourceName] string name, + IResourceBuilder subnet) + { + ArgumentNullException.ThrowIfNull(builder); + ArgumentException.ThrowIfNullOrEmpty(name); + ArgumentNullException.ThrowIfNull(subnet); + + // AGC requires the Gateway API CRDs, so both ingressProfile properties are + // enabled together. These flags drive the preview API version + property + // injection in ConfigureAksInfrastructure. + builder.Resource.GatewayApiEnabled = true; + builder.Resource.ApplicationLoadBalancerEnabled = true; + + // Delegate the subnet to AGC. AKS node-pool subnets are non-delegated, so this + // delegation only applies to user-supplied ALB subnets. + // + // AzureSubnetResource emits a single delegation in its provisioning entity and + // honors only the LAST AzureSubnetServiceDelegationAnnotation on the subnet + // (last write wins). A naive `HasAnnotationOfType<...>()` short-circuit would + // therefore silently swallow our AGC delegation if the caller had already + // delegated the subnet to something else (e.g. Microsoft.NetApp/volumes), and + // the deployment would later fail with an opaque AGC association error. + // + // Instead, only skip when the most recent delegation already targets + // trafficControllers (so multiple AddLoadBalancer calls sharing a subnet stay + // idempotent). Otherwise, append our annotation so it ends up last and AGC is + // the delegation actually emitted. + var existingDelegations = subnet.Resource.Annotations.OfType().ToList(); + var lastDelegation = existingDelegations.Count > 0 ? existingDelegations[^1] : null; + string? displacedDelegationServiceName = null; + if (lastDelegation is null + || !string.Equals(lastDelegation.ServiceName, "Microsoft.ServiceNetworking/trafficControllers", StringComparison.Ordinal)) + { + // Capture the displaced delegation (if any) so the LB pipeline step can warn + // the user at deploy time that their explicit delegation was silently overridden. + // We can't log here because no ILogger is available during model construction; + // the resource's apply-alb-crd pipeline step has access to context.Logger. + if (lastDelegation is not null) + { + displacedDelegationServiceName = lastDelegation.ServiceName; + } + + subnet.WithAnnotation(new AzureSubnetServiceDelegationAnnotation( + "Microsoft.ServiceNetworking/trafficControllers", + "Microsoft.ServiceNetworking/trafficControllers")); + } + + var lb = new AzureKubernetesLoadBalancerResource( + name, + builder.Resource, + subnet.Resource.Id, + subnet.Resource, + displacedDelegationServiceName); + + // Track the LB on the env so ConfigureAksInfrastructure can emit a role + // assignment binding the AKS-auto-created AGC controller identity to the + // user-supplied subnet. Done in both run and publish modes so any future + // run-mode introspection sees a consistent set of LBs; the subsequent + // run-mode early-return below skips the model registration only. + builder.Resource.LoadBalancers.Add(lb); + + // In run mode the AKS environment is not added to the model (see + // AddAzureKubernetesEnvironment), so its aks-get-credentials-{name} + // pipeline step is never registered. Mirror that pattern here so the + // LB's apply-alb-crd-{name} step (which depends on aks-get-credentials) + // is also not registered, avoiding pipeline validation failures. + if (builder.ApplicationBuilder.ExecutionContext.IsRunMode) + { + return builder.ApplicationBuilder.CreateResourceBuilder(lb); + } + + return builder.ApplicationBuilder.AddResource(lb) + .ExcludeFromManifest(); + } + /// /// Enables or disables workload identity on the AKS environment, allowing pods to authenticate /// to Azure services using federated credentials. @@ -516,6 +645,21 @@ private static void ConfigureAksInfrastructure(AzureResourceInfrastructure infra infrastructure.Add(aks); + // Surface the preview-only ingress profile properties for AGC / managed Gateway API. + // We bump to the oldest preview API version that has both gatewayAPI and + // applicationLoadBalancer; the injection itself is reflection-based because the + // Azure.Provisioning.ContainerService types that own these properties are internal. + // The xmldoc on AksPreviewIngressProfileInjector documents the public DefineProperty / + // DefineModelProperty alternatives that were tried and empirically ruled out. + if (aksResource.RequiresPreviewIngressApi) + { + aks.ResourceVersion = "2025-09-02-preview"; + AksPreviewIngressProfileInjector.Inject( + aks, + gatewayApi: aksResource.GatewayApiEnabled, + applicationLoadBalancer: aksResource.ApplicationLoadBalancerEnabled); + } + // ACR pull role assignment for kubelet identity if (aksResource.DefaultContainerRegistry is not null || aksResource.TryGetLastAnnotation(out _)) { @@ -553,6 +697,85 @@ private static void ConfigureAksInfrastructure(AzureResourceInfrastructure infra infrastructure.Add(roleAssignment); } + // AGC ALB controller subnet role assignments. AKS auto-creates a managed identity + // for the AGC ALB add-on (`applicationloadbalancer-{cluster-name}` in the MC_* + // resource group) when `ingressProfile.applicationLoadBalancer.enabled` is set, + // but only auto-grants it permissions on resources inside MC_*. When the user + // supplies an ALB subnet that lives outside MC_* (e.g. in the cluster's parent + // RG), the controller fails with `LinkedAuthorizationFailed` on + // `Microsoft.Network/virtualNetworks/subnets/join/action`. We close that gap by + // emitting a `Network Contributor` role assignment per LB subnet, scoped to the + // subnet, with the principalId read back from the cluster's + // `properties.ingressProfile.applicationLoadBalancer.identity.objectId` output. + // The schema marks that identity property `readOnly`, so AKS owns the lifecycle + // and we just consume it after the cluster is provisioned. + // See https://learn.microsoft.com/en-us/azure/application-gateway/for-containers/quickstart-deploy-application-gateway-for-containers-alb-controller-addon + // for the documented role bindings the addon needs. + if (aksResource.LoadBalancers.Count > 0) + { + // Network Contributor role: 4d97b98b-1d4f-4787-a291-c67834d212e7. Picked + // because it includes `Microsoft.Network/virtualNetworks/subnets/join/action`, + // matching the BYO-deployment guidance for AGC associations. + var networkContributorRoleId = BicepFunction.GetSubscriptionResourceId( + "Microsoft.Authorization/roleDefinitions", + "4d97b98b-1d4f-4787-a291-c67834d212e7"); + + var albAddonPrincipalId = new MemberExpression( + new MemberExpression( + new MemberExpression( + new MemberExpression( + new MemberExpression( + new IdentifierExpression(aks.BicepIdentifier), + "properties"), + "ingressProfile"), + "applicationLoadBalancer"), + "identity"), + "objectId"); + + // Dedupe (vnet, subnet) pairs so multiple LBs sharing a subnet only emit a + // single existing-resource declaration and a single role assignment. + var subnetExistingByKey = new Dictionary(StringComparer.Ordinal); + var assignedSubnets = new HashSet(StringComparer.Ordinal); + + foreach (var lb in aksResource.LoadBalancers) + { + var subnet = lb.SubnetResource + ?? throw new InvalidOperationException($"AzureKubernetesLoadBalancerResource '{lb.Name}' is missing its subnet binding."); + var vnet = subnet.Parent; + + // Reuse the canonical existing-VNet handle so emitted Bicep references + // match the rest of the module and we don't double-declare the resource. + var existingVnet = (VirtualNetwork)vnet.AddAsExistingResource(infrastructure); + + var subnetIdentifier = $"{existingVnet.BicepIdentifier}_{Infrastructure.NormalizeBicepIdentifier(subnet.Name)}_existing"; + if (!subnetExistingByKey.TryGetValue(subnetIdentifier, out var existingSubnet)) + { + existingSubnet = SubnetResource.FromExisting(subnetIdentifier); + existingSubnet.Parent = existingVnet; + existingSubnet.Name = subnet.SubnetName; + infrastructure.Add(existingSubnet); + subnetExistingByKey[subnetIdentifier] = existingSubnet; + } + + if (!assignedSubnets.Add(subnetIdentifier)) + { + continue; + } + + var albSubnetRole = new RoleAssignment($"albSubnetJoin_{Infrastructure.NormalizeBicepIdentifier(lb.Name)}") + { + // GUID name keyed off subnet + cluster + role so reruns are idempotent + // and parallel LBs targeting different subnets don't collide. + Name = BicepFunction.CreateGuid(existingSubnet.Id, aks.Id, networkContributorRoleId), + Scope = new IdentifierExpression(existingSubnet.BicepIdentifier), + RoleDefinitionId = networkContributorRoleId, + PrincipalId = albAddonPrincipalId, + PrincipalType = RoleManagementPrincipalType.ServicePrincipal + }; + infrastructure.Add(albSubnetRole); + } + } + // Outputs infrastructure.Add(new ProvisioningOutput("id", typeof(string)) { Value = aks.Id }); infrastructure.Add(new ProvisioningOutput("name", typeof(string)) { Value = aks.Name }); diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentResource.AksPipeline.cs b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentResource.AksPipeline.cs index 335d47b6d4a..15781036db5 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentResource.AksPipeline.cs +++ b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentResource.AksPipeline.cs @@ -286,6 +286,245 @@ await getCredsTask.FailAsync( } } + /// + /// Applies the AGC ApplicationLoadBalancer custom resource for the supplied + /// into the cluster. Polls the + /// cluster for the azure-alb-external GatewayClass first (it appears once the + /// AGC ALB controller add-on is fully installed), then kubectl applys the CR + /// pointing at the load balancer's delegated subnet. + /// + /// + /// The CR shape is documented at + /// https://learn.microsoft.com/azure/application-gateway/for-containers/quickstart-deploy-application-gateway-for-containers. + /// Example: + /// + /// apiVersion: alb.networking.azure.io/v1 + /// kind: ApplicationLoadBalancer + /// metadata: + /// name: alb-{lb.Name} + /// namespace: default + /// spec: + /// associations: + /// - /subscriptions/.../subnets/{albSubnet} + /// + /// + internal async Task ApplyAlbCrdAsync( + AzureKubernetesLoadBalancerResource lb, + PipelineStepContext context) + { + var applyTask = await context.ReportingStep.CreateTaskAsync( + $"Applying AGC ApplicationLoadBalancer CR for {lb.Name}", + context.CancellationToken).ConfigureAwait(false); + + await using (applyTask.ConfigureAwait(false)) + { + try + { + if (lb.DisplacedDelegationServiceName is { } displaced) + { + // The subnet had an explicit non-trafficControllers service delegation when + // AddLoadBalancer was called. AzureSubnetResource emits only the LAST + // AzureSubnetServiceDelegationAnnotation, so AGC's trafficControllers + // delegation displaced the user's. Warn at deploy time so the user can + // either remove the original delegation or use a separate subnet. + context.Logger.LogWarning( + "AddLoadBalancer overrode an existing service delegation '{DisplacedServiceName}' " + + "on the subnet for AGC load balancer '{LoadBalancerName}' with " + + "'Microsoft.ServiceNetworking/trafficControllers'. AGC requires this delegation; " + + "if you need '{DisplacedServiceName}' to remain, use a separate subnet for the load balancer.", + displaced, lb.Name, displaced); + } + + var subnetId = await ((IValueProvider)lb.SubnetIdReference).GetValueAsync(context.CancellationToken).ConfigureAwait(false); + if (string.IsNullOrEmpty(subnetId)) + { + throw new InvalidOperationException( + $"Could not resolve subnet ID for AGC load balancer '{lb.Name}'."); + } + + var kubeConfigPath = KubernetesEnvironment.KubeConfigPath + ?? throw new InvalidOperationException( + $"Cannot apply AGC ApplicationLoadBalancer CR for '{lb.Name}': " + + $"kubeconfig was not set by aks-get-credentials-{Name}."); + + // Wait for the azure-alb-external GatewayClass to appear. The AGC ALB + // controller add-on installs it asynchronously, so polling is required + // even after the AKS cluster reports Succeeded. 10-minute budget matches + // the E2E test budget in KubernetesGatewayTlsDeploymentTests.cs. + await WaitForAzureAlbGatewayClassAsync( + kubeConfigPath, context.Logger, TimeSpan.FromMinutes(10), + context.CancellationToken).ConfigureAwait(false); + + // Apply the ApplicationLoadBalancer CR via kubectl apply -f - using stdin + // so we don't need a temp file. JSON is a valid YAML subset for kubectl. + var manifest = + $$""" + { + "apiVersion": "alb.networking.azure.io/v1", + "kind": "ApplicationLoadBalancer", + "metadata": { "name": "{{lb.AlbName}}", "namespace": "{{AzureKubernetesLoadBalancerResource.AlbNamespace}}" }, + "spec": { "associations": ["{{subnetId}}"] } + } + """; + + var applyArgs = $"apply --kubeconfig \"{kubeConfigPath}\" -n \"{AzureKubernetesLoadBalancerResource.AlbNamespace}\" -f -"; + + // Buffer stderr (and the tail of stdout, since kubectl sometimes writes + // structured errors to stdout when --output is not requested) so we can + // surface the real failure cause in the thrown exception. Without this, + // the only signal a caller gets is the exit code, which makes RBAC, + // missing-CRD, and admission-webhook errors very hard to diagnose. Cap + // the buffer to keep a pathological controller from blowing up the + // exception message; 4 KB is plenty for the multi-line "the server + // could not find the requested resource" / "forbidden" / validation + // messages kubectl emits. + const int kubectlErrorCaptureBytes = 4 * 1024; + var errorCapture = new StringBuilder(); + + void CaptureLine(string line) + { + if (string.IsNullOrEmpty(line) || errorCapture.Length >= kubectlErrorCaptureBytes) + { + return; + } + + var remaining = kubectlErrorCaptureBytes - errorCapture.Length; + if (line.Length + 1 > remaining) + { + errorCapture.Append(line, 0, Math.Max(0, remaining - 1)); + } + else + { + errorCapture.AppendLine(line); + } + } + + var applySpec = new ProcessSpec("kubectl") + { + Arguments = applyArgs, + StandardInputContent = manifest, + InheritEnv = true, + ThrowOnNonZeroReturnCode = false, + OnOutputData = line => + { + context.Logger.LogDebug("kubectl: {Line}", line); + CaptureLine(line); + }, + OnErrorData = line => + { + context.Logger.LogDebug("kubectl: {Line}", line); + CaptureLine(line); + } + }; + + var (applyResultTask, applyDisposable) = ProcessUtil.Run(applySpec); + int applyExitCode; + await using (applyDisposable.ConfigureAwait(false)) + { + var result = await applyResultTask.WaitAsync(context.CancellationToken).ConfigureAwait(false); + applyExitCode = result.ExitCode; + } + + if (applyExitCode != 0) + { + var capturedOutput = errorCapture.ToString().TrimEnd(); + var detail = string.IsNullOrEmpty(capturedOutput) + ? "kubectl produced no diagnostic output; re-run the deploy with debug logging enabled to see kubectl's stderr." + : capturedOutput; + + throw new InvalidOperationException( + $"kubectl apply for ApplicationLoadBalancer '{lb.AlbName}' failed (exit code {applyExitCode}).{Environment.NewLine}{detail}"); + } + + context.Logger.LogInformation( + "Applied ApplicationLoadBalancer '{AlbName}' in namespace '{AlbNamespace}' bound to subnet '{SubnetId}'", + lb.AlbName, AzureKubernetesLoadBalancerResource.AlbNamespace, subnetId); + + context.Summary.Add( + $"☸ ALB {lb.Name}", + new MarkdownString($"**{lb.AlbName}** in `{AzureKubernetesLoadBalancerResource.AlbNamespace}` (subnet `{subnetId}`)")); + + await applyTask.SucceedAsync( + $"AGC ApplicationLoadBalancer '{lb.AlbName}' applied", + context.CancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + await applyTask.FailAsync( + $"Failed to apply AGC ApplicationLoadBalancer for {lb.Name}: {ex.Message}", + context.CancellationToken).ConfigureAwait(false); + throw; + } + } + } + + /// + /// Polls kubectl get gatewayclass azure-alb-external until it succeeds or the + /// timeout elapses. The azure-alb-external GatewayClass is installed by the + /// AGC ALB controller add-on (ingressProfile.applicationLoadBalancer.enabled), + /// but provisioning is asynchronous — the AKS resource may report Succeeded before the + /// add-on's CRDs land in the cluster. + /// + private static async Task WaitForAzureAlbGatewayClassAsync( + string kubeConfigPath, + ILogger logger, + TimeSpan timeout, + CancellationToken cancellationToken) + { + var deadline = DateTime.UtcNow + timeout; + var pollInterval = TimeSpan.FromSeconds(10); + var attempt = 0; + + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + + attempt++; + // Silent OnErrorData/OnOutputData is intentional for this poll loop: + // every probe before the GatewayClass lands prints a "NotFound" stderr line + // that would just be noise. The terminal failure mode (timeout) below + // emits an actionable error that points at the AKS preview feature flags. + var (resultTask, disposable) = ProcessUtil.Run(new ProcessSpec("kubectl") + { + Arguments = $"get gatewayclass azure-alb-external --kubeconfig \"{kubeConfigPath}\" --no-headers", + ThrowOnNonZeroReturnCode = false, + InheritEnv = true, + OnOutputData = _ => { }, + OnErrorData = _ => { } + }); + + int exitCode; + await using (disposable.ConfigureAwait(false)) + { + var result = await resultTask.WaitAsync(cancellationToken).ConfigureAwait(false); + exitCode = result.ExitCode; + } + + if (exitCode == 0) + { + logger.LogInformation( + "GatewayClass 'azure-alb-external' is available (after {Attempts} probe(s))", + attempt); + return; + } + + if (DateTime.UtcNow >= deadline) + { + throw new InvalidOperationException( + "Timed out waiting for the 'azure-alb-external' GatewayClass to appear in the cluster. " + + "Ensure the AKS preview features 'Microsoft.ContainerService/AKSGatewayAPIPreview' and " + + "'Microsoft.ContainerService/AKSAppGatewayContainersPreview' are registered on the subscription, " + + "and that the AGC ALB controller add-on (ingressProfile.applicationLoadBalancer) finished installing."); + } + + logger.LogDebug( + "GatewayClass 'azure-alb-external' not yet available (attempt {Attempt}); retrying in {Delay}s", + attempt, pollInterval.TotalSeconds); + + await Task.Delay(pollInterval, cancellationToken).ConfigureAwait(false); + } + } + private static string FindAzCli() { var azPath = PathLookupHelper.FindFullPathFromPath("az"); diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentResource.cs b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentResource.cs index d43b6cdee3e..6b0e3de146e 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentResource.cs +++ b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesEnvironmentResource.cs @@ -164,4 +164,37 @@ public AzureKubernetesEnvironmentResource( /// Gets or sets the default container registry auto-created for this AKS environment. /// internal AzureContainerRegistryResource? DefaultContainerRegistry { get; set; } + + /// + /// Gets the load balancer resources registered against this AKS environment via + /// . Used by + /// the Bicep emission to synthesize per-LB role assignments granting the + /// AKS-auto-created AGC controller identity permission to join each LB subnet. + /// + internal List LoadBalancers { get; } = []; + + /// + /// Gets or sets whether the AKS managed Gateway API installation is enabled on the + /// cluster. Toggled internally by ; + /// not exposed as a public extension because it's only useful in combination with the + /// AGC ALB controller add-on () today. + /// + internal bool GatewayApiEnabled { get; set; } + + /// + /// Gets or sets whether the Azure Application Gateway for Containers (AGC) ALB + /// controller add-on is enabled on the cluster. Toggled internally by + /// . + /// + internal bool ApplicationLoadBalancerEnabled { get; set; } + + /// + /// Whether the cluster needs to be emitted using a preview Bicep API version because + /// it depends on ingressProfile.gatewayAPI or ingressProfile.applicationLoadBalancer, + /// neither of which is in any stable AKS API version yet (latest stable + /// 2026-01-01 doesn't have them; gatewayAPI first appears in + /// 2025-08-02-preview, applicationLoadBalancer in + /// 2025-09-02-preview). + /// + internal bool RequiresPreviewIngressApi => GatewayApiEnabled || ApplicationLoadBalancerEnabled; } diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesIngressExtensions.cs b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesIngressExtensions.cs index 14d0759bdbc..2f7f3618820 100644 --- a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesIngressExtensions.cs +++ b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesIngressExtensions.cs @@ -132,4 +132,77 @@ public static IResourceBuilder AddHelmChart( var k8sEnvBuilder = builder.ApplicationBuilder.CreateResourceBuilder(builder.Resource.KubernetesEnvironment); return k8sEnvBuilder.AddHelmChart(name, chartReference, chartVersion); } + + /// + /// Routes a Kubernetes through the supplied + /// Azure Application Gateway for Containers (AGC) . + /// + /// The gateway resource builder. + /// The AGC load balancer to route through. + /// A reference to the for chaining. + /// + /// Writes the AGC routing annotations (alb.networking.azure.io/alb-name and + /// alb.networking.azure.io/alb-namespace) onto the rendered Gateway and defaults + /// the GatewayClass to azure-alb-external when one has not already been set via + /// WithGatewayClass(...). + /// + /// + /// + /// var lb = aks.AddLoadBalancer("lb", albSubnet); + /// var gateway = aks.AddGateway("public").WithLoadBalancer(lb); + /// + /// + [AspireExport(Description = "Routes a Kubernetes Gateway through an AGC ApplicationLoadBalancer")] + public static IResourceBuilder WithLoadBalancer( + this IResourceBuilder builder, + IResourceBuilder loadBalancer) + { + ArgumentNullException.ThrowIfNull(builder); + ArgumentNullException.ThrowIfNull(loadBalancer); + + var lb = loadBalancer.Resource; + // AGC discovers the target ApplicationLoadBalancer via these two annotations on + // the Gateway. See: + // https://learn.microsoft.com/azure/application-gateway/for-containers/quickstart-deploy-application-gateway-for-containers + builder.Resource.GatewayAnnotations["alb.networking.azure.io/alb-name"] = + ReferenceExpression.Create($"{lb.AlbName}"); + builder.Resource.GatewayAnnotations["alb.networking.azure.io/alb-namespace"] = + ReferenceExpression.Create($"{AzureKubernetesLoadBalancerResource.AlbNamespace}"); + + // Default to the AGC GatewayClass only when one isn't already set so an explicit + // WithGatewayClass(...) call before WithLoadBalancer(...) wins. + builder.Resource.GatewayClassName ??= ReferenceExpression.Create($"azure-alb-external"); + return builder; + } + + /// + /// Routes a Kubernetes through the supplied + /// Azure Application Gateway for Containers (AGC) . + /// + /// The ingress resource builder. + /// The AGC load balancer to route through. + /// A reference to the for chaining. + /// + /// Writes the AGC routing annotations (alb.networking.azure.io/alb-name and + /// alb.networking.azure.io/alb-namespace) onto the rendered Ingress and defaults + /// the IngressClass to azure-alb-external when one has not already been set via + /// WithIngressClass(...). + /// + [AspireExport("withLoadBalancerOnIngress", MethodName = "withLoadBalancer", Description = "Routes a Kubernetes Ingress through an AGC ApplicationLoadBalancer")] + public static IResourceBuilder WithLoadBalancer( + this IResourceBuilder builder, + IResourceBuilder loadBalancer) + { + ArgumentNullException.ThrowIfNull(builder); + ArgumentNullException.ThrowIfNull(loadBalancer); + + var lb = loadBalancer.Resource; + builder.Resource.IngressAnnotations["alb.networking.azure.io/alb-name"] = + ReferenceExpression.Create($"{lb.AlbName}"); + builder.Resource.IngressAnnotations["alb.networking.azure.io/alb-namespace"] = + ReferenceExpression.Create($"{AzureKubernetesLoadBalancerResource.AlbNamespace}"); + + builder.Resource.IngressClassName ??= ReferenceExpression.Create($"azure-alb-external"); + return builder; + } } diff --git a/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesLoadBalancerResource.cs b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesLoadBalancerResource.cs new file mode 100644 index 00000000000..ae8ac3c4daa --- /dev/null +++ b/src/Aspire.Hosting.Azure.Kubernetes/AzureKubernetesLoadBalancerResource.cs @@ -0,0 +1,142 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#pragma warning disable ASPIREPIPELINES001 // PipelineStepAnnotation/PipelineStep are evaluation-only +#pragma warning disable ASPIREAZURE003 // AzureSubnetResource is evaluation-only + +using Aspire.Hosting.ApplicationModel; +using Aspire.Hosting.Pipelines; + +namespace Aspire.Hosting.Azure.Kubernetes; + +/// +/// Represents a single Azure Application Gateway for Containers (AGC) +/// ApplicationLoadBalancer Kubernetes custom resource ( +/// alb.networking.azure.io/v1) bound to a delegated subnet. +/// +/// +/// +/// Each AGC ApplicationLoadBalancer is capped at 5 frontends, so larger +/// applications create multiple load balancer resources via repeated calls to +/// and associate +/// gateways/ingresses with a specific load balancer using +/// . +/// +/// +/// The resource registers a per-LB apply-alb-crd-{name} pipeline step that +/// runs after AKS credentials are fetched and before Helm chart preparation. The +/// step polls the cluster for the azure-alb-external GatewayClass (installed +/// by the AGC ALB controller add-on), then kubectl applys the +/// ApplicationLoadBalancer custom resource pointing at the supplied subnet. +/// +/// +public sealed class AzureKubernetesLoadBalancerResource : + Resource, + IResourceWithParent +{ + /// + /// Initializes a new instance of the class. + /// + /// The name of the load balancer resource. Used to derive the in-cluster + /// ApplicationLoadBalancer name (alb-{name}) referenced by gateway/ingress annotations. + /// The parent AKS environment that owns this load balancer. + /// Reference to the resource ID of the delegated subnet + /// this load balancer associates with. Resolved at deployment time by the + /// apply-alb-crd-{name} pipeline step and emitted into the spec.associations + /// field of the ApplicationLoadBalancer CR. + /// The Aspire subnet resource that backs this load balancer. + /// If non-, the name of an + /// existing subnet service delegation that + /// silently overrode with the AGC trafficControllers delegation. The pipeline step logs + /// a warning at deploy time so the user can investigate. + /// + /// All deploy-time state (, ) + /// is taken as a constructor parameter rather than via object-initializer setters because the + /// constructor eagerly registers a whose deferred action + /// dereferences these fields. Using = default! properties with { get; set; } would + /// create a window where forgetting to set the property results in a runtime + /// at deploy with no compile-time signal. Compare + /// AzureContainerAppResource, AzureContainerRegistryResource, and + /// AzureAppServiceWebSiteResource, which follow the same pattern. + /// + internal AzureKubernetesLoadBalancerResource( + string name, + AzureKubernetesEnvironmentResource parent, + BicepOutputReference subnetIdReference, + Aspire.Hosting.Azure.AzureSubnetResource subnetResource, + string? displacedDelegationServiceName = null) + : base(name) + { + ArgumentNullException.ThrowIfNull(parent); + ArgumentNullException.ThrowIfNull(subnetIdReference); + ArgumentNullException.ThrowIfNull(subnetResource); + + Parent = parent; + SubnetIdReference = subnetIdReference; + SubnetResource = subnetResource; + DisplacedDelegationServiceName = displacedDelegationServiceName; + + // Register the per-LB pipeline step that applies the ApplicationLoadBalancer + // CR into the cluster after credentials are available. Using a factory lambda + // so the step is materialized lazily by the pipeline configuration phase. + Annotations.Add(new PipelineStepAnnotation(_ => + { + var step = new PipelineStep + { + Name = $"apply-alb-crd-{Name}", + Description = $"Applies the AGC ApplicationLoadBalancer CR for {Name}.", + Action = ctx => Parent.ApplyAlbCrdAsync(this, ctx), + // Kubeconfig must be set first. + DependsOnSteps = [$"aks-get-credentials-{Parent.Name}"], + // Helm prepare can then reference the LB by name in gateway/ingress annotations. + RequiredBySteps = [$"prepare-{Parent.KubernetesEnvironment.Name}"] + }; + + return Task.FromResult>([step]); + })); + } + + /// + public AzureKubernetesEnvironmentResource Parent { get; } + + /// + /// Reference to the resource ID of the delegated subnet that this load balancer + /// associates with. Resolved at deployment time and emitted into the + /// spec.associations field of the ApplicationLoadBalancer CR. + /// + internal BicepOutputReference SubnetIdReference { get; } + + /// + /// The Aspire subnet resource that backs this load balancer. Captured so the AKS + /// environment's Bicep emission can synthesize a per-LB role assignment granting + /// the AKS-auto-created AGC controller identity the + /// Microsoft.Network/virtualNetworks/subnets/join/action permission on the + /// subnet (via Network Contributor). Without this, AKS only auto-grants the + /// AGC identity permissions inside the cluster's MC_* node resource group, so + /// any user-supplied subnet outside that RG fails with LinkedAuthorizationFailed + /// when the controller tries to create the AGC association. + /// + internal Aspire.Hosting.Azure.AzureSubnetResource SubnetResource { get; } + + /// + /// The name of an existing subnet service delegation that + /// silently overrode with the AGC trafficControllers delegation, or + /// if no override occurred. The deploy-time pipeline step logs a warning when this is set so the + /// user is alerted that their original delegation will not be emitted into Bicep. + /// + internal string? DisplacedDelegationServiceName { get; } + + /// + /// The in-cluster name of the ApplicationLoadBalancer CR. This is the value + /// AGC expects in the alb.networking.azure.io/alb-name annotation on + /// gateway/ingress resources. + /// + internal string AlbName => $"alb-{Name}"; + + /// + /// The Kubernetes namespace the ApplicationLoadBalancer CR is created in. + /// Currently fixed to default; a future WithNamespace extension can + /// surface this when multi-tenant clusters need isolation. + /// + internal static string AlbNamespace => "default"; +} diff --git a/src/Aspire.Hosting.Kubernetes/KubernetesEnvironmentResource.cs b/src/Aspire.Hosting.Kubernetes/KubernetesEnvironmentResource.cs index 627ce06a41f..985f45f166f 100644 --- a/src/Aspire.Hosting.Kubernetes/KubernetesEnvironmentResource.cs +++ b/src/Aspire.Hosting.Kubernetes/KubernetesEnvironmentResource.cs @@ -994,6 +994,22 @@ private static async Task DiscoverFqdnAndBootstrapTlsAsync( var gatewayName = gateway.Name.ToKubernetesResourceName(); var secretName = await ResolveExpressionAsync(secretNameExpr, context.CancellationToken).ConfigureAwait(false); + // Pre-create a placeholder bootstrap TLS secret BEFORE waiting for the Gateway + // address. Some controllers (notably Azure Application Gateway for Containers) + // refuse to program a Gateway whose HTTPS listener references a non-existent + // Secret — they log "Secret 'X' not found" and stop reconciling. That creates a + // deadlock with the FQDN discovery flow, which itself waits for the Gateway to + // be programmed. Uploading a self-signed placeholder up front breaks the + // chicken-and-egg: the Gateway can program with the placeholder cert, get an + // address, and we then patch the listener hostname so cert-manager can replace + // the placeholder with a real certificate. + await EnsureBootstrapTlsSecretAsync( + secretName, + hostname: "bootstrap.invalid", + @namespace, + environment, + context).ConfigureAwait(false); + // Poll for the Gateway's assigned hostname address. // We use -o json and parse the full status to select Hostname-type addresses, // since some controllers return IP addresses which are not valid for TLS hostnames. @@ -1085,88 +1101,120 @@ await TransferGatewayFieldOwnership( gatewayName, @namespace, environment, context).ConfigureAwait(false); } - // Check if bootstrap TLS secret already exists - var checkArgs = $"get secret {secretName} --namespace {@namespace}"; - if (environment.KubeConfigPath is not null) - { - checkArgs += $" --kubeconfig \"{environment.KubeConfigPath}\""; - } + // Bootstrap TLS secret was already pre-created above. Nothing further to do + // here — cert-manager will replace it with a real certificate once the + // listener hostname is detected on the Gateway. + } + } - var (checkResult, checkDisposable) = ProcessUtil.Run(new ProcessSpec("kubectl") - { - Arguments = checkArgs, - ThrowOnNonZeroReturnCode = false, - InheritEnv = true, - OnOutputData = _ => { }, - OnErrorData = _ => { } - }); + /// + /// Creates a self-signed bootstrap TLS secret with the given hostname as the CN/SAN + /// if it doesn't already exist. Used by the FQDN discovery flow to break the + /// chicken-and-egg between Gateway controllers (which need the secret to program the + /// Gateway) and FQDN discovery (which needs the Gateway to be programmed). + /// + private static async Task EnsureBootstrapTlsSecretAsync( + string secretName, + string hostname, + string @namespace, + KubernetesEnvironmentResource environment, + PipelineStepContext context) + { + var checkArgs = $"get secret {secretName} --namespace {@namespace}"; + if (environment.KubeConfigPath is not null) + { + checkArgs += $" --kubeconfig \"{environment.KubeConfigPath}\""; + } - await using (checkDisposable.ConfigureAwait(false)) + var (checkResult, checkDisposable) = ProcessUtil.Run(new ProcessSpec("kubectl") + { + Arguments = checkArgs, + ThrowOnNonZeroReturnCode = false, + InheritEnv = true, + OnOutputData = _ => { }, + OnErrorData = _ => { } + }); + + await using (checkDisposable.ConfigureAwait(false)) + { + var result = await checkResult.WaitAsync(context.CancellationToken).ConfigureAwait(false); + if (result.ExitCode == 0) { - var result = await checkResult.WaitAsync(context.CancellationToken).ConfigureAwait(false); - if (result.ExitCode == 0) - { - context.Logger.LogInformation("TLS secret '{SecretName}' already exists, skipping bootstrap.", secretName); - continue; - } + context.Logger.LogInformation("TLS secret '{SecretName}' already exists, skipping bootstrap.", secretName); + return; } + } - // Create a bootstrap self-signed cert with the discovered FQDN - context.Logger.LogInformation("Creating bootstrap TLS secret '{SecretName}' for '{Hostname}'.", secretName, discoveredFqdn); + context.Logger.LogInformation("Creating bootstrap TLS secret '{SecretName}' for '{Hostname}'.", secretName, hostname); - using var ecdsa = ECDsa.Create(ECCurve.NamedCurves.nistP256); - var certRequest = new CertificateRequest($"CN={discoveredFqdn}", ecdsa, HashAlgorithmName.SHA256); - var sanBuilder = new SubjectAlternativeNameBuilder(); - sanBuilder.AddDnsName(discoveredFqdn); - certRequest.CertificateExtensions.Add(sanBuilder.Build()); - using var cert = certRequest.CreateSelfSigned(DateTimeOffset.UtcNow, DateTimeOffset.UtcNow.AddDays(1)); + using var ecdsa = ECDsa.Create(ECCurve.NamedCurves.nistP256); + var certRequest = new CertificateRequest($"CN={hostname}", ecdsa, HashAlgorithmName.SHA256); + var sanBuilder = new SubjectAlternativeNameBuilder(); + sanBuilder.AddDnsName(hostname); + certRequest.CertificateExtensions.Add(sanBuilder.Build()); + using var cert = certRequest.CreateSelfSigned(DateTimeOffset.UtcNow, DateTimeOffset.UtcNow.AddDays(1)); - var certPem = cert.ExportCertificatePem(); - var keyPem = ecdsa.ExportECPrivateKeyPem(); + var certPem = cert.ExportCertificatePem(); + var keyPem = ecdsa.ExportECPrivateKeyPem(); - var tempDir = Directory.CreateTempSubdirectory(".aspire-tls-discovery"); - try + var tempDir = Directory.CreateTempSubdirectory(".aspire-tls-discovery"); + try + { + var certPath = Path.Combine(tempDir.FullName, "tls.crt"); + var keyPath = Path.Combine(tempDir.FullName, "tls.key"); + await File.WriteAllTextAsync(certPath, certPem, context.CancellationToken).ConfigureAwait(false); + await File.WriteAllTextAsync(keyPath, keyPem, context.CancellationToken).ConfigureAwait(false); + + var createArgs = $"create secret tls {secretName} --cert=\"{certPath}\" --key=\"{keyPath}\" --namespace {@namespace}"; + if (environment.KubeConfigPath is not null) { - var certPath = Path.Combine(tempDir.FullName, "tls.crt"); - var keyPath = Path.Combine(tempDir.FullName, "tls.key"); - await File.WriteAllTextAsync(certPath, certPem, context.CancellationToken).ConfigureAwait(false); - await File.WriteAllTextAsync(keyPath, keyPem, context.CancellationToken).ConfigureAwait(false); + createArgs += $" --kubeconfig \"{environment.KubeConfigPath}\""; + } - var createArgs = $"create secret tls {secretName} --cert=\"{certPath}\" --key=\"{keyPath}\" --namespace {@namespace}"; - if (environment.KubeConfigPath is not null) - { - createArgs += $" --kubeconfig \"{environment.KubeConfigPath}\""; - } + var (createResult, createDisposable) = ProcessUtil.Run(new ProcessSpec("kubectl") + { + Arguments = createArgs, + ThrowOnNonZeroReturnCode = false, + InheritEnv = true, + OnOutputData = line => context.Logger.LogDebug("{Line}", line), + OnErrorData = line => context.Logger.LogDebug("{Line}", line) + }); - var (createResult, createDisposable) = ProcessUtil.Run(new ProcessSpec("kubectl") + await using (createDisposable.ConfigureAwait(false)) + { + var createExitResult = await createResult.WaitAsync(context.CancellationToken).ConfigureAwait(false); + if (createExitResult.ExitCode != 0) { - Arguments = createArgs, - ThrowOnNonZeroReturnCode = false, - InheritEnv = true, - OnOutputData = line => context.Logger.LogDebug("{Line}", line), - OnErrorData = line => context.Logger.LogDebug("{Line}", line) - }); - - await using (createDisposable.ConfigureAwait(false)) + context.Logger.LogWarning("Failed to create bootstrap TLS secret '{SecretName}' (exit code {ExitCode}).", secretName, createExitResult.ExitCode); + } + else { - var createExitResult = await createResult.WaitAsync(context.CancellationToken).ConfigureAwait(false); - if (createExitResult.ExitCode != 0) - { - context.Logger.LogWarning("Failed to create bootstrap TLS secret '{SecretName}' (exit code {ExitCode}).", secretName, createExitResult.ExitCode); - } - else - { - context.Logger.LogInformation( - "Bootstrap TLS secret '{SecretName}' created for '{Hostname}'. " + - "cert-manager will replace this with a real certificate once the hostname is detected on the Gateway listener.", - secretName, discoveredFqdn); - } + context.Logger.LogInformation( + "Bootstrap TLS secret '{SecretName}' created for '{Hostname}'. " + + "cert-manager will replace this with a real certificate once the hostname is detected on the Gateway listener.", + secretName, hostname); } } - finally - { - try { tempDir.Delete(recursive: true); } catch { } - } + } + finally + { + DeleteTempDirSafely(tempDir, context.Logger); + } + } + + // Best-effort cleanup for kubectl/cert temp directories. Swallowing OperationCanceledException + // here would mask shutdown signals; instead narrow to file-system failures (a transient + // antivirus lock or permission issue) and surface them at debug level so the next deploy + // attempt - or `aspire run` with verbose logging - can diagnose if leakage occurs. + private static void DeleteTempDirSafely(DirectoryInfo tempDir, ILogger logger) + { + try + { + tempDir.Delete(recursive: true); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + logger.LogDebug(ex, "Failed to delete temporary directory '{TempDir}'.", tempDir.FullName); } } @@ -1456,7 +1504,7 @@ private static async Task TransferGatewayFieldOwnership( } finally { - try { tempDir.Delete(recursive: true); } catch { } + DeleteTempDirSafely(tempDir, context.Logger); } } catch (System.Text.Json.JsonException ex) @@ -1560,7 +1608,7 @@ private static async Task BootstrapTlsSecretsAsync( } finally { - try { tempDir.Delete(recursive: true); } catch { } + DeleteTempDirSafely(tempDir, context.Logger); } } } diff --git a/src/Aspire.Hosting.Kubernetes/KubernetesHelmChartExtensions.cs b/src/Aspire.Hosting.Kubernetes/KubernetesHelmChartExtensions.cs index 53b9a0c6272..1a4c71c67b6 100644 --- a/src/Aspire.Hosting.Kubernetes/KubernetesHelmChartExtensions.cs +++ b/src/Aspire.Hosting.Kubernetes/KubernetesHelmChartExtensions.cs @@ -73,6 +73,16 @@ public static IResourceBuilder AddHelmChart( var environment = builder.Resource; var resource = new KubernetesHelmChartResource(name, environment, chartReference, chartVersion); + // Helm chart installation is a publish/deploy-time concern only. In run mode the + // parent KubernetesEnvironmentResource isn't added to the model (see + // AddKubernetesEnvironment), so any helm-install step that depends on + // helm-deploy-{env.Name} would fail step validation with a missing-dependency error. + // Mirror AddIngress/AddGateway and skip model registration entirely in run mode. + if (builder.ApplicationBuilder.ExecutionContext.IsRunMode) + { + return builder.ApplicationBuilder.CreateResourceBuilder(resource); + } + var chartBuilder = builder.ApplicationBuilder.AddResource(resource); chartBuilder.WithAnnotation(new PipelineStepAnnotation(_ => @@ -210,6 +220,62 @@ public static IResourceBuilder WithDestroy( return builder; } + /// + /// Opts the Helm chart in to helm upgrade --install --force-conflicts. When set, + /// Helm's server-side apply forcibly takes over any fields owned by another field + /// manager instead of failing with a conflict. + /// + /// The Helm chart resource builder. + /// The resource builder for chaining. + /// + /// + /// This is most commonly needed for charts whose templates ship admission webhooks + /// (cert-manager, kyverno, gatekeeper, opa-gatekeeper, etc.) on clusters where another + /// admission controller — such as the AKS admissionsenforcer field manager + /// installed by the Azure Policy add-on or Deployment Safeguards — mutates the webhook + /// configuration after install. Helm's Server-Side Apply (used by default for charts + /// that opt in, including cert-manager) refuses to overwrite fields owned by another + /// field manager. Without --force-conflicts, the next helm upgrade fails + /// with a "conflict with admissionsenforcer" error on the webhook's + /// namespaceSelector (or similar). + /// See + /// Deployment Safeguards in AKS + /// and + /// Server-Side Apply conflicts + /// for background. + /// + /// + /// Unlike the deprecated --force / --force-replace (which delete and + /// recreate the resource and are incompatible with Server-Side Apply), + /// --force-conflicts is non-destructive — it only changes which field manager + /// owns the conflicting field. No resources are deleted or recreated. This flag is + /// also distinct from Helm's --take-ownership, which transfers ownership of an + /// entire resource between Helm releases and does not address field-level conflicts. + /// + /// + /// Requires Helm v3.18 or later (the version that introduced --force-conflicts + /// for helm upgrade --install). Older Helm versions fail with + /// Error: unknown flag: --force-conflicts. + /// + /// + /// + /// + /// // cert-manager on AKS clusters with Azure Policy / Deployment Safeguards enabled. + /// k8s.AddHelmChart("cert-manager", "oci://quay.io/jetstack/charts/cert-manager", "v1.18.2") + /// .WithHelmValue("crds.enabled", "true") + /// .WithForceConflicts(); + /// + /// + [AspireExport("withHelmChartForceConflicts", Description = "Passes --force-conflicts to helm upgrade --install for this chart")] + public static IResourceBuilder WithForceConflicts( + this IResourceBuilder builder) + { + ArgumentNullException.ThrowIfNull(builder); + + builder.Resource.ForceConflicts = true; + return builder; + } + private static async Task InstallHelmChartAsync( PipelineStepContext context, KubernetesEnvironmentResource environment, @@ -232,6 +298,27 @@ private static async Task InstallHelmChartAsync( arguments.Append(CultureInfo.InvariantCulture, $" --version {chart.ChartVersion}"); + if (chart.ForceConflicts) + { + // --force-conflicts tells helm's server-side apply to forcibly take over fields + // owned by other field managers instead of failing with a conflict. Required + // for charts whose admission webhooks are mutated by the AKS admissionsenforcer + // / Azure Policy add-on after install — without it, helm's SSA fails on + // .webhooks[*].namespaceSelector. Non-destructive (no resource recreate). + // Equivalent to `kubectl apply --force-conflicts` and distinct from + // --take-ownership (which transfers helm release ownership) and the + // deprecated --force / --force-replace (which delete + recreate resources + // and are incompatible with SSA). + // See KubernetesHelmChartExtensions.WithForceConflicts for the full rationale. + // + // --server-side is REQUIRED alongside --force-conflicts: helm only registers + // the --force-conflicts flag in server-side-apply mode. Without --server-side, + // helm rejects the unknown flag with "Error: unknown flag: --force-conflicts" + // before it even attempts to install the chart. Both flags arrived together + // in helm v3.18. + arguments.Append(" --server-side --force-conflicts"); + } + if (environment.KubeConfigPath is not null) { arguments.Append(CultureInfo.InvariantCulture, $" --kubeconfig {QuoteArg(environment.KubeConfigPath)}"); diff --git a/src/Aspire.Hosting.Kubernetes/KubernetesHelmChartResource.cs b/src/Aspire.Hosting.Kubernetes/KubernetesHelmChartResource.cs index a451480ff93..92efbb11ce6 100644 --- a/src/Aspire.Hosting.Kubernetes/KubernetesHelmChartResource.cs +++ b/src/Aspire.Hosting.Kubernetes/KubernetesHelmChartResource.cs @@ -106,4 +106,12 @@ public KubernetesHelmChartResource( /// are not removed automatically. /// internal bool DestroyOnUninstall { get; set; } + + /// + /// Gets a value indicating whether helm upgrade --install should be invoked with + /// the --force-conflicts flag. Set via + /// . + /// Defaults to . + /// + internal bool ForceConflicts { get; set; } } diff --git a/tests/Aspire.Deployment.EndToEnd.Tests/AksAzureKubernetesEnvironmentGatewayDeploymentTests.cs b/tests/Aspire.Deployment.EndToEnd.Tests/AksAzureKubernetesEnvironmentGatewayDeploymentTests.cs new file mode 100644 index 00000000000..b51ca811fd2 --- /dev/null +++ b/tests/Aspire.Deployment.EndToEnd.Tests/AksAzureKubernetesEnvironmentGatewayDeploymentTests.cs @@ -0,0 +1,343 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Aspire.Cli.Tests.Utils; +using Aspire.Deployment.EndToEnd.Tests.Helpers; +using Hex1b.Automation; +using Xunit; + +namespace Aspire.Deployment.EndToEnd.Tests; + +/// +/// End-to-end test for the AddAzureKubernetesEnvironment + AddLoadBalancer + +/// AddGateway story. The Aspire deploy pipeline provisions the AKS cluster, ACR, VNet, +/// the AGC ingress profile (via the Bicep change in this PR), the AGC ALB controller add-on, +/// the ApplicationLoadBalancer CR, and the Gateway API Gateway + HTTPRoute +/// resources. The test then waits for the AGC data plane to assign an FQDN to the gateway and +/// verifies the API service is reachable over plain HTTP via that FQDN. +/// TLS issuance via cert-manager is intentionally NOT exercised here; that is covered by +/// . +/// +public sealed class AksAzureKubernetesEnvironmentGatewayDeploymentTests(ITestOutputHelper output) +{ + // Provisioning AKS + AGC takes ~15 min, deploy + verify a few more — give 60 min headroom. + private static readonly TimeSpan s_testTimeout = TimeSpan.FromMinutes(60); + + [Fact] + public async Task DeployApiWithGatewayToAzureKubernetesEnvironment() + { + using var cts = new CancellationTokenSource(s_testTimeout); + using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource( + cts.Token, TestContext.Current.CancellationToken); + var cancellationToken = linkedCts.Token; + + await DeployApiWithGatewayToAzureKubernetesEnvironmentCore(cancellationToken); + } + + private async Task DeployApiWithGatewayToAzureKubernetesEnvironmentCore(CancellationToken cancellationToken) + { + var subscriptionId = AzureAuthenticationHelpers.TryGetSubscriptionId(); + if (string.IsNullOrEmpty(subscriptionId)) + { + Assert.Skip("Azure subscription not configured. Set ASPIRE_DEPLOYMENT_TEST_SUBSCRIPTION."); + } + + if (!AzureAuthenticationHelpers.IsAzureAuthAvailable()) + { + if (DeploymentE2ETestHelpers.IsRunningInCI) + { + Assert.Fail("Azure authentication not available in CI. Check OIDC configuration."); + } + else + { + Assert.Skip("Azure authentication not available. Run 'az login' to authenticate."); + } + } + + var workspace = TemporaryWorkspace.Create(output); + var startTime = DateTime.UtcNow; + var deploymentUrls = new Dictionary(); + var resourceGroupName = DeploymentE2ETestHelpers.GenerateResourceGroupName("aksgw"); + var projectName = "AksGatewayApi"; + + output.WriteLine($"Test: {nameof(DeployApiWithGatewayToAzureKubernetesEnvironment)}"); + output.WriteLine($"Project Name: {projectName}"); + output.WriteLine($"Resource Group: {resourceGroupName}"); + output.WriteLine($"Subscription: {subscriptionId[..8]}..."); + output.WriteLine($"Workspace: {workspace.WorkspaceRoot.FullName}"); + + try + { + using var terminal = DeploymentE2ETestHelpers.CreateTestTerminal(); + var pendingRun = terminal.RunAsync(cancellationToken); + + var counter = new SequenceCounter(); + var auto = new Hex1bTerminalAutomator(terminal, defaultTimeout: TimeSpan.FromSeconds(500)); + + // Step 1: Prepare environment + output.WriteLine("Step 1: Preparing environment..."); + await auto.PrepareEnvironmentAsync(workspace, counter); + + // Step 2: Set up CLI environment + await auto.InstallCurrentBuildAspireCliAsync(counter, output); + + // Step 3: Create starter project (no Redis — we just need an API service to expose). + output.WriteLine("Step 3: Creating Aspire starter project..."); + await auto.AspireNewAsync(projectName, counter, useRedisCache: false); + + // Step 4: Navigate to project directory + output.WriteLine("Step 4: Navigating to project directory..."); + await auto.TypeAsync($"cd {projectName}"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter); + + // Step 5: Add Aspire.Hosting.Azure.Kubernetes package + output.WriteLine("Step 5: Adding Azure Kubernetes hosting package..."); + await auto.TypeAsync("aspire add Aspire.Hosting.Azure.Kubernetes"); + await auto.EnterAsync(); + await auto.WaitForAspireAddCompletionAsync(counter); + + // Step 6: Patch AppHost.cs with AddAzureKubernetesEnvironment + AddLoadBalancer + AddGateway. + // + // The patched snippet mirrors playground/AksDemo/AksDemo.AppHost/AppHost.cs but + // intentionally drops cert-manager / WithTls / cluster-issuer wiring — we want the + // raw Bicep-provisions-AGC + gateway-attaches-to-LB code paths covered without + // dragging Let's Encrypt into this test. + // + // We rely on the starter template exposing `var apiService = builder.AddProject_ApiService>("apiservice")`. + // The webfrontend line is left untouched; we don't route to it from the gateway. + var projectDir = Path.Combine(workspace.WorkspaceRoot.FullName, projectName); + var appHostDir = Path.Combine(projectDir, $"{projectName}.AppHost"); + var appHostFilePath = Path.Combine(appHostDir, "AppHost.cs"); + + output.WriteLine($"Step 6: Modifying AppHost.cs at: {appHostFilePath}"); + + var content = File.ReadAllText(appHostFilePath); + + // The AGC ingress profile + ApplicationLoadBalancer + Gateway/HTTPRoute pieces that + // this PR adds. Inject before builder.Build().Run();. Use Standard_D2as_v5 to match + // the other AKS deployment tests' SKU/region quota story. + const string buildRunPattern = "builder.Build().Run();"; + const string replacement = """ +// VNet layout chosen to avoid the AKS default service CIDR (10.0.0.0/16): +// 10.100.0.0/16 - vnet +// 10.100.0.0/22 - aks node pool subnet +// 10.100.4.0/24 - AGC frontend subnet (delegated to ServiceNetworking by AddLoadBalancer) +var vnet = builder.AddAzureVirtualNetwork("vnet", "10.100.0.0/16"); +var aksSubnet = vnet.AddSubnet("aks-nodes", "10.100.0.0/22"); +var albSubnet = vnet.AddSubnet("alb-public", "10.100.4.0/24"); + +var aks = builder.AddAzureKubernetesEnvironment("aks") + .WithSubnet(aksSubnet) + .WithSystemNodePool("Standard_D2as_v5"); +aks.AddNodePool("workload", "Standard_D2as_v5", minCount: 1, maxCount: 3); + +// AddLoadBalancer creates the AGC ApplicationLoadBalancer CR, delegates the frontend subnet +// to Microsoft.ServiceNetworking, and (per this PR) ensures the AGC managed identity gets +// Network Contributor on the subnet so the controller can program the data plane. +var publicLb = aks.AddLoadBalancer("public", albSubnet); + +// Gateway with a single route that points at / on the apiService. WithLoadBalancer +// stamps the alb.networking.azure.io association annotations and defaults the +// gatewayClassName to "azure-alb-external". Routing "/" (Prefix) so any path the +// starter template's apiservice exposes (/, /weatherforecast) flows through. +aks.AddGateway("api-gw") + .WithLoadBalancer(publicLb) + .WithRoute("/", apiService.GetEndpoint("http")); + +builder.Build().Run(); +"""; + + content = content.Replace(buildRunPattern, replacement); + + // Fail loudly if the starter template ever drops the literal we patch on: + // without this guard, `Replace` silently returns the original string and the + // test would deploy a stock starter app and report "success" without + // exercising any of the AGC / Gateway / LoadBalancer code paths under test. + Assert.Contains(buildRunPattern, content); + + // Required pragmas for the new (still experimental) AGC + pipeline surface. + const string pragmaBlock = + "#pragma warning disable ASPIREPIPELINES001\n" + + "#pragma warning disable ASPIRECOMPUTE003\n" + + "#pragma warning disable ASPIREAZURE003\n"; + + if (!content.Contains("#pragma warning disable ASPIREPIPELINES001")) + { + content = pragmaBlock + content; + } + + File.WriteAllText(appHostFilePath, content); + output.WriteLine("Modified AppHost.cs with AddAzureKubernetesEnvironment + AddLoadBalancer + AddGateway"); + + // Step 7: Navigate to AppHost project directory + output.WriteLine("Step 7: Navigating to AppHost directory..."); + await auto.TypeAsync($"cd {projectName}.AppHost"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter); + + // Step 8: Set environment variables for deployment. + // - Unset ASPIRE_PLAYGROUND to avoid conflicts. + // - Set Azure location to westus3 (where we have Standard_D2as_v5 capacity, matching + // the rest of the AKS deployment tests). + // - Set AZURE__RESOURCEGROUP to use our unique resource group name so the finally + // block can clean it up. + await auto.TypeAsync($"unset ASPIRE_PLAYGROUND && export AZURE__LOCATION=westus3 && export AZURE__RESOURCEGROUP={resourceGroupName}"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter); + + // Step 9: Deploy. + // --clear-cache prevents reuse of any cached location/RG from a previous local run. + output.WriteLine("Step 9: Starting AKS deployment (provisioning AKS + AGC takes 10-15 min)..."); + await auto.TypeAsync("aspire deploy --clear-cache"); + await auto.EnterAsync(); + await auto.WaitForPipelineSuccessAsync(timeout: TimeSpan.FromMinutes(35)); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromMinutes(2)); + + // Step 10: Get AKS credentials for the auto-provisioned cluster. + output.WriteLine("Step 10: Getting AKS credentials..."); + await auto.TypeAsync($"AKS_NAME=$(az aks list -g {resourceGroupName} --query '[0].name' -o tsv) && " + + $"az aks get-credentials -g {resourceGroupName} -n $AKS_NAME --overwrite-existing"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(60)); + + // Step 11: Wait for all pods to be ready across namespaces (the helm release that + // aspire deploy installs lands in its own namespace named after the project). + output.WriteLine("Step 11: Waiting for pods to be ready..."); + await auto.TypeAsync("kubectl wait --for=condition=ready pod --all --all-namespaces --timeout=300s 2>/dev/null || true"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromMinutes(6)); + + await auto.TypeAsync("kubectl get pods --all-namespaces && kubectl get gateway --all-namespaces"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(30)); + + // Step 12: Discover the namespace where the api-gw gateway lives (set by helm release + // namespace, which aspire deploy chooses based on the project name). + output.WriteLine("Step 12: Discovering gateway namespace..."); + await auto.TypeAsync("NS=$(kubectl get gateway --all-namespaces -o jsonpath='{range .items[?(@.metadata.name==\"api-gw\")]}{.metadata.namespace}{end}') && echo \"Namespace: $NS\""); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(30)); + + // Step 13: Wait for AGC to assign an FQDN to the gateway. AGC data plane provisioning + // can take 5-10 min on a fresh cluster the first time the ALB association lands. + output.WriteLine("Step 13: Waiting for AGC to assign gateway FQDN (up to 15 min)..."); + await auto.TypeAsync( + "OK=0; for i in $(seq 1 90); do " + + "FQDN=$(kubectl get gateway api-gw -n $NS -o jsonpath='{.status.addresses[0].value}' 2>/dev/null); " + + "[ -n \"$FQDN\" ] && echo \"Gateway FQDN: $FQDN\" && OK=1 && break; " + + "echo \"Attempt $i: waiting for AGC FQDN...\"; sleep 10; done; " + + "[ \"$OK\" = \"1\" ] || { echo 'FAIL: gateway never received AGC FQDN'; exit 1; }"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromMinutes(16)); + + // Step 14: Verify the API responds over the AGC FQDN. Because AGC programs the data + // plane asynchronously after the FQDN is published, retry for a couple of minutes. + // /weatherforecast is the actual API endpoint exposed by the starter template + // apiservice — the gateway is wired to "/" (Prefix) so the path flows through to it. + output.WriteLine("Step 14: Verifying http:///weatherforecast returns 200..."); + await auto.TypeAsync( + "FQDN=$(kubectl get gateway api-gw -n $NS -o jsonpath='{.status.addresses[0].value}') && " + + "echo \"Testing: http://$FQDN/weatherforecast\" && " + + "OK=0; for i in $(seq 1 30); do sleep 5; " + + "S=$(curl -so /dev/null -w '%{http_code}' -m 10 http://$FQDN/weatherforecast 2>/dev/null); " + + "[ \"$S\" = \"200\" ] && echo \"HTTP $S OK\" && OK=1 && break; " + + "echo \"Attempt $i: HTTP $S\"; done; " + + "[ \"$OK\" = \"1\" ] || { echo 'FAIL: gateway never returned 200 via AGC FQDN'; exit 1; }"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromMinutes(4)); + + // Step 15: Sanity-check via port-forward. This isolates "app is healthy" from + // "AGC routing is healthy" and matches the pattern used by AksBlazorRedis. + output.WriteLine("Step 15: Verifying apiservice via port-forward..."); + await auto.TypeAsync("kubectl port-forward svc/apiservice-service 18080:8080 -n $NS > /dev/null 2>&1 &"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); + + await auto.TypeAsync("sleep 3"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); + + // /weatherforecast is the actual API endpoint exposed by the starter template + // apiservice in non-Development. Fail explicitly if all retries are exhausted. + await auto.TypeAsync( + "OK=0; for i in $(seq 1 10); do sleep 3 && " + + "curl -sf http://localhost:18080/weatherforecast -o /dev/null -w '%{http_code}' && " + + "echo ' OK' && OK=1 && break; done; " + + "[ \"$OK\" = \"1\" ] || { echo 'FAIL: apiservice unreachable via port-forward'; exit 1; }"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(60)); + + await auto.TypeAsync("kill %1 2>/dev/null; true"); + await auto.EnterAsync(); + await auto.WaitForSuccessPromptAsync(counter, TimeSpan.FromSeconds(10)); + + // Step 16: Tear down via the Aspire pipeline. + output.WriteLine("Step 16: Destroying deployment..."); + await auto.AspireDestroyAsync(counter); + + await auto.TypeAsync("exit"); + await auto.EnterAsync(); + + await pendingRun; + + var duration = DateTime.UtcNow - startTime; + output.WriteLine($"Deployment completed in {duration}"); + + DeploymentReporter.ReportDeploymentSuccess( + nameof(DeployApiWithGatewayToAzureKubernetesEnvironment), + resourceGroupName, + deploymentUrls, + duration); + + output.WriteLine("✅ Test passed!"); + } + catch (Exception ex) + { + var duration = DateTime.UtcNow - startTime; + output.WriteLine($"❌ Test failed after {duration}: {ex.Message}"); + + DeploymentReporter.ReportDeploymentFailure( + nameof(DeployApiWithGatewayToAzureKubernetesEnvironment), + resourceGroupName, + ex.Message, + ex.StackTrace); + + throw; + } + finally + { + // Fire-and-forget RG delete; the hourly cleanup workflow handles any misses. + output.WriteLine($"Triggering cleanup of resource group: {resourceGroupName}"); + TriggerCleanupResourceGroup(resourceGroupName); + } + } + + private void TriggerCleanupResourceGroup(string resourceGroupName) + { + using var process = new System.Diagnostics.Process + { + StartInfo = new System.Diagnostics.ProcessStartInfo + { + FileName = "az", + Arguments = $"group delete --name {resourceGroupName} --yes --no-wait", + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true + } + }; + + try + { + process.Start(); + output.WriteLine($"Cleanup triggered for resource group: {resourceGroupName}"); + DeploymentReporter.ReportCleanupStatus(resourceGroupName, success: true, "Cleanup triggered (fire-and-forget)"); + } + catch (Exception ex) + { + output.WriteLine($"Failed to trigger cleanup: {ex.Message}"); + DeploymentReporter.ReportCleanupStatus(resourceGroupName, success: false, ex.Message); + } + } +} diff --git a/tests/Aspire.Hosting.Azure.Kubernetes.Tests/AzureKubernetesEnvironmentExtensionsTests.cs b/tests/Aspire.Hosting.Azure.Kubernetes.Tests/AzureKubernetesEnvironmentExtensionsTests.cs index 4fdfe112cea..dfdb3d1f18b 100644 --- a/tests/Aspire.Hosting.Azure.Kubernetes.Tests/AzureKubernetesEnvironmentExtensionsTests.cs +++ b/tests/Aspire.Hosting.Azure.Kubernetes.Tests/AzureKubernetesEnvironmentExtensionsTests.cs @@ -3,11 +3,13 @@ #pragma warning disable ASPIREAZURE003 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. #pragma warning disable ASPIRECOMPUTE003 // Type is for evaluation purposes only +#pragma warning disable ASPIREPIPELINES001 // PipelineStepAnnotation is evaluation-only using System.Runtime.CompilerServices; using Aspire.Hosting.ApplicationModel; using Aspire.Hosting.Azure.Kubernetes; using Aspire.Hosting.Kubernetes; +using Aspire.Hosting.Pipelines; using Aspire.Hosting.Utils; using Microsoft.Extensions.DependencyInjection; @@ -436,4 +438,87 @@ public async Task WithSystemNodePool_BicepReflectsCustomVmSize() [UnsafeAccessor(UnsafeAccessorKind.Method, Name = "ExecuteBeforeStartHooksAsync")] private static extern Task ExecuteBeforeStartHooksAsync(DistributedApplication app, CancellationToken cancellationToken); + + [Fact] + public async Task AddLoadBalancer_BicepEnablesIngressProfileAndUsesPreviewApi() + { + using var builder = TestDistributedApplicationBuilder.Create(DistributedApplicationOperation.Publish); + + var vnet = builder.AddAzureVirtualNetwork("vnet", "10.0.0.0/16"); + var aksSubnet = vnet.AddSubnet("aksnodes", "10.0.0.0/22"); + var albSubnet = vnet.AddSubnet("alb", "10.0.4.0/24"); + + var aks = builder.AddAzureKubernetesEnvironment("aks").WithSubnet(aksSubnet); + aks.AddLoadBalancer("lb", albSubnet); + + var manifest = await AzureManifestUtils.GetManifestWithBicep(aks.Resource); + await Verify(manifest.BicepText, extension: "bicep"); + } + + [Fact] + public void AddLoadBalancer_AppliesSubnetDelegation() + { + using var builder = TestDistributedApplicationBuilder.Create(DistributedApplicationOperation.Publish); + + var vnet = builder.AddAzureVirtualNetwork("vnet", "10.0.0.0/16"); + var albSubnet = vnet.AddSubnet("alb", "10.0.4.0/24"); + + var aks = builder.AddAzureKubernetesEnvironment("aks"); + aks.AddLoadBalancer("lb", albSubnet); + + Assert.True(aks.Resource.GatewayApiEnabled); + Assert.True(aks.Resource.ApplicationLoadBalancerEnabled); + Assert.True(aks.Resource.RequiresPreviewIngressApi); + + // AGC requires the subnet be delegated to Microsoft.ServiceNetworking/trafficControllers. + Assert.True(albSubnet.Resource.TryGetLastAnnotation(out var delegation)); + Assert.Equal("Microsoft.ServiceNetworking/trafficControllers", delegation!.ServiceName); + } + + [Fact] + public void AddLoadBalancer_RegistersPerLBPipelineStep() + { + using var builder = TestDistributedApplicationBuilder.Create(DistributedApplicationOperation.Publish); + + var vnet = builder.AddAzureVirtualNetwork("vnet", "10.0.0.0/16"); + var albSubnet = vnet.AddSubnet("alb", "10.0.4.0/24"); + + var aks = builder.AddAzureKubernetesEnvironment("aks"); + var lb = aks.AddLoadBalancer("lb", albSubnet); + + Assert.Same(aks.Resource, lb.Resource.Parent); + Assert.Equal("alb-lb", lb.Resource.AlbName); + Assert.True(lb.Resource.TryGetAnnotationsOfType(out var stepAnnotations)); + Assert.Single(stepAnnotations); + } + + [Fact] + public void AddLoadBalancer_MultipleLBs_AllStepsRegistered() + { + using var builder = TestDistributedApplicationBuilder.Create(DistributedApplicationOperation.Publish); + + var vnet = builder.AddAzureVirtualNetwork("vnet", "10.0.0.0/16"); + var alb1 = vnet.AddSubnet("alb1", "10.0.4.0/24"); + var alb2 = vnet.AddSubnet("alb2", "10.0.5.0/24"); + + var aks = builder.AddAzureKubernetesEnvironment("aks"); + var lb1 = aks.AddLoadBalancer("lb1", alb1); + var lb2 = aks.AddLoadBalancer("lb2", alb2); + + Assert.NotSame(lb1.Resource, lb2.Resource); + Assert.Equal("alb-lb1", lb1.Resource.AlbName); + Assert.Equal("alb-lb2", lb2.Resource.AlbName); + + // Each LB owns its own pipeline step annotation. + Assert.True(lb1.Resource.TryGetAnnotationsOfType(out var lb1Steps)); + Assert.True(lb2.Resource.TryGetAnnotationsOfType(out var lb2Steps)); + Assert.Single(lb1Steps); + Assert.Single(lb2Steps); + + // Both LBs share the subnet/delegation requirements but the annotation + // is applied idempotently per subnet (one delegation per subnet). + Assert.True(alb1.Resource.TryGetLastAnnotation(out _)); + Assert.True(alb2.Resource.TryGetLastAnnotation(out _)); + } + } diff --git a/tests/Aspire.Hosting.Azure.Kubernetes.Tests/AzureKubernetesIngressTests.cs b/tests/Aspire.Hosting.Azure.Kubernetes.Tests/AzureKubernetesIngressTests.cs index e36e0f45078..ea9922f6a8e 100644 --- a/tests/Aspire.Hosting.Azure.Kubernetes.Tests/AzureKubernetesIngressTests.cs +++ b/tests/Aspire.Hosting.Azure.Kubernetes.Tests/AzureKubernetesIngressTests.cs @@ -59,4 +59,74 @@ public void AksAddGateway_HasCorrectParent() Assert.IsType(gateway.Resource); Assert.IsType(gateway.Resource.Parent); } + + [Fact] + public async Task WithLoadBalancer_OnGateway_AnnotatesAndDefaultsClass() + { + var builder = TestDistributedApplicationBuilder.Create(DistributedApplicationOperation.Publish); + var vnet = builder.AddAzureVirtualNetwork("vnet", "10.0.0.0/16"); + var albSubnet = vnet.AddSubnet("alb", "10.0.4.0/24"); + + var aks = builder.AddAzureKubernetesEnvironment("aks"); + var lb = aks.AddLoadBalancer("lb1", albSubnet); + + var gateway = aks.AddGateway("public").WithLoadBalancer(lb); + + Assert.NotNull(gateway.Resource.GatewayClassName); + var resolvedClass = await gateway.Resource.GatewayClassName!.GetValueAsync(default); + Assert.Equal("azure-alb-external", resolvedClass); + + Assert.True(gateway.Resource.GatewayAnnotations.TryGetValue("alb.networking.azure.io/alb-name", out var albNameRef)); + Assert.Equal("alb-lb1", await albNameRef!.GetValueAsync(default)); + + Assert.True(gateway.Resource.GatewayAnnotations.TryGetValue("alb.networking.azure.io/alb-namespace", out var albNsRef)); + Assert.Equal("default", await albNsRef!.GetValueAsync(default)); + } + + [Fact] + public async Task WithLoadBalancer_OnIngress_AnnotatesAndDefaultsClass() + { + var builder = TestDistributedApplicationBuilder.Create(DistributedApplicationOperation.Publish); + var vnet = builder.AddAzureVirtualNetwork("vnet", "10.0.0.0/16"); + var albSubnet = vnet.AddSubnet("alb", "10.0.4.0/24"); + + var aks = builder.AddAzureKubernetesEnvironment("aks"); + var lb = aks.AddLoadBalancer("lb1", albSubnet); + + var ingress = aks.AddIngress("public").WithLoadBalancer(lb); + + Assert.NotNull(ingress.Resource.IngressClassName); + var resolvedClass = await ingress.Resource.IngressClassName!.GetValueAsync(default); + Assert.Equal("azure-alb-external", resolvedClass); + + Assert.True(ingress.Resource.IngressAnnotations.TryGetValue("alb.networking.azure.io/alb-name", out var albNameRef)); + Assert.Equal("alb-lb1", await albNameRef!.GetValueAsync(default)); + + Assert.True(ingress.Resource.IngressAnnotations.TryGetValue("alb.networking.azure.io/alb-namespace", out var albNsRef)); + Assert.Equal("default", await albNsRef!.GetValueAsync(default)); + } + + [Fact] + public async Task WithLoadBalancer_RespectsExplicitGatewayClass() + { + var builder = TestDistributedApplicationBuilder.Create(DistributedApplicationOperation.Publish); + var vnet = builder.AddAzureVirtualNetwork("vnet", "10.0.0.0/16"); + var albSubnet = vnet.AddSubnet("alb", "10.0.4.0/24"); + + var aks = builder.AddAzureKubernetesEnvironment("aks"); + var lb = aks.AddLoadBalancer("lb1", albSubnet); + + // Explicit class set BEFORE WithLoadBalancer is preserved; AGC annotations + // are still applied so AGC can still discover the LB. + var gateway = aks.AddGateway("public") + .WithGatewayClass("custom-class") + .WithLoadBalancer(lb); + + Assert.NotNull(gateway.Resource.GatewayClassName); + var resolvedClass = await gateway.Resource.GatewayClassName!.GetValueAsync(default); + Assert.Equal("custom-class", resolvedClass); + + Assert.True(gateway.Resource.GatewayAnnotations.ContainsKey("alb.networking.azure.io/alb-name")); + Assert.True(gateway.Resource.GatewayAnnotations.ContainsKey("alb.networking.azure.io/alb-namespace")); + } } diff --git a/tests/Aspire.Hosting.Azure.Kubernetes.Tests/Snapshots/AzureKubernetesEnvironmentExtensionsTests.AddLoadBalancer_BicepEnablesIngressProfileAndUsesPreviewApi.verified.bicep b/tests/Aspire.Hosting.Azure.Kubernetes.Tests/Snapshots/AzureKubernetesEnvironmentExtensionsTests.AddLoadBalancer_BicepEnablesIngressProfileAndUsesPreviewApi.verified.bicep new file mode 100644 index 00000000000..79bfa97b7a6 --- /dev/null +++ b/tests/Aspire.Hosting.Azure.Kubernetes.Tests/Snapshots/AzureKubernetesEnvironmentExtensionsTests.AddLoadBalancer_BicepEnablesIngressProfileAndUsesPreviewApi.verified.bicep @@ -0,0 +1,103 @@ +@description('The location for the resource(s) to be deployed.') +param location string = resourceGroup().location + +param subnetId string + +param acrName string + +param vnet_outputs_name string + +resource aks 'Microsoft.ContainerService/managedClusters@2025-09-02-preview' = { + name: take('aks-${uniqueString(resourceGroup().id)}', 63) + tags: { + 'aspire-resource-name': 'aks' + } + location: location + properties: { + dnsPrefix: 'aks-dns' + agentPoolProfiles: [ + { + name: 'system' + count: 1 + vmSize: 'Standard_D2s_v5' + vnetSubnetID: subnetId + osType: 'Linux' + maxCount: 3 + minCount: 1 + enableAutoScaling: true + mode: 'System' + } + ] + oidcIssuerProfile: { + enabled: true + } + networkProfile: { + networkPlugin: 'azure' + } + securityProfile: { + workloadIdentity: { + enabled: true + } + } + ingressProfile: { + gatewayAPI: { + installation: 'Standard' + } + applicationLoadBalancer: { + enabled: true + } + } + } + sku: { + name: 'Base' + tier: 'Free' + } + identity: { + type: 'SystemAssigned' + } +} + +resource acr 'Microsoft.ContainerRegistry/registries@2025-04-01' existing = { + name: acrName +} + +resource acrPullRole 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(acr.id, aks.id, subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '7f951dda-4ed3-4680-a7ca-43fe172d538d')) + properties: { + principalId: aks.properties.identityProfile.kubeletidentity.objectId + roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '7f951dda-4ed3-4680-a7ca-43fe172d538d') + principalType: 'ServicePrincipal' + } + scope: acr +} + +resource vnet 'Microsoft.Network/virtualNetworks@2025-05-01' existing = { + name: vnet_outputs_name +} + +resource vnet_alb_existing 'Microsoft.Network/virtualNetworks/subnets@2025-05-01' existing = { + name: 'alb' + parent: vnet +} + +resource albSubnetJoin_lb 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(vnet_alb_existing.id, aks.id, subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '4d97b98b-1d4f-4787-a291-c67834d212e7')) + properties: { + principalId: aks.properties.ingressProfile.applicationLoadBalancer.identity.objectId + roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '4d97b98b-1d4f-4787-a291-c67834d212e7') + principalType: 'ServicePrincipal' + } + scope: vnet_alb_existing +} + +output id string = aks.id + +output name string = aks.name + +output clusterFqdn string = aks.properties.fqdn + +output oidcIssuerUrl string = aks.properties.oidcIssuerProfile.issuerURL + +output kubeletIdentityObjectId string = aks.properties.identityProfile.kubeletidentity.objectId + +output nodeResourceGroup string = aks.properties.nodeResourceGroup \ No newline at end of file diff --git a/tests/Aspire.Hosting.Kubernetes.Tests/KubernetesHelmChartTests.cs b/tests/Aspire.Hosting.Kubernetes.Tests/KubernetesHelmChartTests.cs index c2632f086d7..7b345683449 100644 --- a/tests/Aspire.Hosting.Kubernetes.Tests/KubernetesHelmChartTests.cs +++ b/tests/Aspire.Hosting.Kubernetes.Tests/KubernetesHelmChartTests.cs @@ -346,6 +346,48 @@ public void WithDestroy_ThrowsOnNullBuilder() ((IResourceBuilder)null!).WithDestroy()); } + [Fact] + public void WithForceConflicts_DefaultsToFalse() + { + var builder = TestDistributedApplicationBuilder.Create(DistributedApplicationOperation.Publish); + var k8s = builder.AddKubernetesEnvironment("env"); + + var chart = k8s.AddHelmChart("test", "oci://example.com/chart", "1.0.0"); + + Assert.False(chart.Resource.ForceConflicts); + } + + [Fact] + public void WithForceConflicts_OptsIn() + { + var builder = TestDistributedApplicationBuilder.Create(DistributedApplicationOperation.Publish); + var k8s = builder.AddKubernetesEnvironment("env"); + + var chart = k8s.AddHelmChart("test", "oci://example.com/chart", "1.0.0") + .WithForceConflicts(); + + Assert.True(chart.Resource.ForceConflicts); + } + + [Fact] + public void WithForceConflicts_ReturnsBuilderForChaining() + { + var builder = TestDistributedApplicationBuilder.Create(DistributedApplicationOperation.Publish); + var k8s = builder.AddKubernetesEnvironment("env"); + + var chart = k8s.AddHelmChart("test", "oci://example.com/chart", "1.0.0"); + var returned = chart.WithForceConflicts(); + + Assert.Same(chart, returned); + } + + [Fact] + public void WithForceConflicts_ThrowsOnNullBuilder() + { + Assert.Throws(() => + ((IResourceBuilder)null!).WithForceConflicts()); + } + [Fact] public async Task PipelineStepFactory_WithoutDestroy_ProducesOnlyInstallStep() {