Use UserDefinedRouting for AKS when running with Private Networking

src/deploy-cromwell-on-azure/Configuration.cs

@@ -31,6 +31,8 @@ public class Configuration : UserAccessibleConfiguration
 
     public abstract class UserAccessibleConfiguration
     {
+        // https://learn.microsoft.com/azure/aks/private-clusters?tabs=azure-portal#configure-a-private-dns-zone CUSTOM_PRIVATE_DNS_ZONE_RESOURCE_ID
+        public string AksPrivateDnsZoneResourceId { get; set; }
         public string AksNodeResourceGroupName { get; set; }
         public string IdentityResourceId { get; set; }
         public string AzureCloudName { get; set; } = AzureCloudConfig.DefaultAzureCloudName;
@@ -44,9 +46,9 @@ public abstract class UserAccessibleConfiguration
         public string VmSubnetAddressSpace { get; set; } = "10.1.0.0/24"; // 10.1.0.0 - 10.1.0.255, 256 IPs
         public string PostgreSqlSubnetAddressSpace { get; set; } = "10.1.1.0/24"; // 10.1.1.0 - 10.1.1.255, 256 IPs
         // Address space for kubernetes system services, must not overlap with any subnet.
-        public string KubernetesServiceCidr = "10.1.4.0/22"; // 10.1.4.0 -> 10.1.7.255, 1024 IPs
-        public string KubernetesDnsServiceIP = "10.1.4.10";
-        public string KubernetesDockerBridgeCidr = "172.17.0.1/16"; // 172.17.0.0 - 172.17.255.255, 65536 IPs
+        public string KubernetesServiceCidr { get; set; } = "10.1.4.0/22"; // 10.1.4.0 -> 10.1.7.255, 1024 IPs
+        public string KubernetesDnsServiceIP { get; set; } = "10.1.4.10";
+        public string KubernetesDockerBridgeCidr { get; set; } = "172.17.0.1/16"; // 172.17.0.0 - 172.17.255.255, 65536 IPs
         public string BatchNodesSubnetAddressSpace { get; set; } = "10.1.128.0/17"; // 10.1.128.0 - 10.1.255.255, 32768 IPs
 
         public string ResourceGroupName { get; set; }
@@ -74,6 +76,9 @@ public abstract class UserAccessibleConfiguration
         public string PostgreSqlSubnetName { get; set; }
         public string BatchSubnetName { get; set; }
         public bool? PrivateNetworking { get; set; } = null;
+        // https://learn.microsoft.com/azure/aks/egress-outboundtype#outbound-type-of-userdefinedrouting
+        // https://learn.microsoft.com/azure/aks/egress-udr
+        public bool? UserDefinedRouting { get; set; } = null;
         public string Tags { get; set; } = null;
         public string BatchNodesSubnetId { get; set; } = null;
         public bool? DisableBatchNodesPublicIpAddress { get; set; } = null;

src/deploy-cromwell-on-azure/Deployer.cs

@@ -933,7 +933,18 @@ private async Task<ContainerServiceManagedClusterResource> ProvisionManagedClust
                     EnablePrivateClusterPublicFqdn = false
                 };
 
+                if (!string.IsNullOrWhiteSpace(configuration.AksPrivateDnsZoneResourceId))
+                {
+                    cluster.ApiServerAccessProfile.PrivateDnsZone = configuration.AksPrivateDnsZoneResourceId;
+                }
+
                 cluster.PublicNetworkAccess = ContainerServicePublicNetworkAccess.Disabled;
+
+                if (configuration.UserDefinedRouting == true)
+                {
+                    cluster.NetworkProfile ??= new();
+                    cluster.NetworkProfile.OutboundType = ContainerServiceOutboundType.UserDefinedRouting;
+                }
             }
 
             return await Execute(
@@ -2212,6 +2223,8 @@ void ValidateHelmInstall(string helmPath, string featureName)
             ThrowIfProvidedForUpdate(configuration.CrossSubscriptionAKSDeployment, nameof(configuration.CrossSubscriptionAKSDeployment));
             ThrowIfProvidedForUpdate(configuration.ApplicationInsightsAccountName, nameof(configuration.ApplicationInsightsAccountName));
             ThrowIfProvidedForUpdate(configuration.PrivateNetworking, nameof(configuration.PrivateNetworking));
+            ThrowIfProvidedForUpdate(configuration.AksPrivateDnsZoneResourceId, nameof(configuration.AksPrivateDnsZoneResourceId));
+            ThrowIfProvidedForUpdate(configuration.UserDefinedRouting, nameof(configuration.UserDefinedRouting));
             ThrowIfProvidedForUpdate(configuration.VnetName, nameof(configuration.VnetName));
             ThrowIfProvidedForUpdate(configuration.VnetResourceGroupName, nameof(configuration.VnetResourceGroupName));
             ThrowIfProvidedForUpdate(configuration.SubnetName, nameof(configuration.SubnetName));