chore(ci): add e2e tests for cloud distros

.github/bundles/rke2/uds-bundle.yaml

@@ -43,6 +43,20 @@ packages:
     optionalComponents:
       - metrics-server
     overrides:
+      istio-admin-gateway:
+        gateway:
+          values:
+            - path: service.annotations
+              value:
+                service.beta.kubernetes.io/aws-load-balancer-scheme: "internet-facing"
+                service.beta.kubernetes.io/aws-load-balancer-target-node-labels: "kubernetes.io/os=linux"
+      istio-tenant-gateway:
+        gateway:
+          values:
+            - path: service.annotations
+              value:
+                service.beta.kubernetes.io/aws-load-balancer-scheme: "internet-facing"
+                service.beta.kubernetes.io/aws-load-balancer-target-node-labels: "kubernetes.io/os=linux"
       velero:
         velero:
           variables:

.github/test-infra/aws/rke2/data.tf

@@ -10,7 +10,7 @@ data "aws_vpc" "vpc" {
 
 data "aws_subnet" "rke2_ci_subnet" {
   vpc_id            = data.aws_vpc.vpc.id
-  availability_zone = "${var.region}c"
+  availability_zone = "${var.region}a"
 
   filter {
     name   = "tag:Name"

.github/test-infra/aws/rke2/iam.tf

@@ -77,6 +77,19 @@ data "aws_iam_policy_document" "aws_ccm" {
   }
 }
 
+data "local_file" "helm_template" {
+  filename = "./scripts/helmchart-template.yaml"
+}
+
+data "http" "aws-lb-controller-iam" {
+  url = "https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/v2.12.0/docs/install/iam_policy_us-gov.json"
+}
+resource "aws_iam_role_policy" "aws-lb-controller" {
+  name = "${local.cluster_name}-lb-controller"
+  role = aws_iam_role.rke2_server.id
+  policy = data.http.aws-lb-controller-iam.response_body
+}
+
 resource "aws_iam_role_policy" "s3_token" {
   name   = "${local.cluster_name}-server-token"
   role   = aws_iam_role.rke2_server.id

.github/test-infra/aws/rke2/main.tf

@@ -26,6 +26,8 @@ locals {
     ccm_external                = true,
     token_bucket                = module.statestore.bucket,
     token_object                = module.statestore.token_object
+    cluster_name                = local.tags.cluster_name
+    helm_chart_template         = file("./scripts/helmchart-template.yaml")
   }
 }
 
@@ -95,7 +97,7 @@ resource "aws_instance" "rke2_ci_control_plane_node" {
   associate_public_ip_address = true
 
   root_block_device {
-    volume_size = 100
+    volume_size = 250
   }
 
   tags = merge(local.tags, { "kubernetes.io/cluster/${local.cluster_name}" = "owned" })
@@ -107,15 +109,16 @@ resource "aws_instance" "rke2_ci_agent_node" {
   ami                         = data.aws_ami.rhel_rke2.image_id
   instance_type               = var.agent_instance_type
   key_name                    = aws_key_pair.control_plane_key_pair.key_name
-  user_data                   = templatefile("${path.module}/scripts/user_data.sh", merge(local.userdata, { BOOTSTRAP_IP = aws_instance.rke2_ci_bootstrap_node.private_ip }))
+  user_data                   = templatefile("${path.module}/scripts/user_data.sh", merge(local.userdata, { BOOTSTRAP_IP = aws_instance.rke2_ci_bootstrap_node.private_ip, AGENT_NODE = true }))
   subnet_id                   = data.aws_subnet.rke2_ci_subnet.id
   user_data_replace_on_change = true
   iam_instance_profile        = aws_iam_instance_profile.rke2_server.name
   vpc_security_group_ids      = [aws_security_group.rke2_ci_node_sg.id]
   associate_public_ip_address = true
+  availability_zone           = "${var.region}a"
 
   root_block_device {
-    volume_size = 100
+    volume_size = 250
   }
 
   tags = merge(local.tags, { "kubernetes.io/cluster/${local.cluster_name}" = "owned" })

.github/test-infra/aws/rke2/scripts/get-kubeconfig.sh

@@ -27,7 +27,7 @@ done
 mkdir -p ~/.kube
 
 # Copy kubectl from cluster node
-ssh -o StrictHostKeyChecking=no -i key.pem ${node_user}@${bootstrap_ip} "mkdir -p /home/${node_user}/.kube && sudo cp /etc/rancher/rke2/rke2.yaml /home/${node_user}/.kube/config && sudo chown ${node_user} /home/${node_user}/.kube/config"  > /dev/null
+ssh -o StrictHostKeyChecking=no -i key.pem ${node_user}@${bootstrap_ip} "mkdir -p /home/${node_user}/.kube && sudo cp /etc/rancher/rke2/rke2.yaml /home/${node_user}/.kube/config && sudo chown ${node_user} /home/${node_user}/.kube/config" > /dev/null
 scp -o StrictHostKeyChecking=no -i key.pem ${node_user}@${bootstrap_ip}:/home/${node_user}/.kube/config ./rke2-config > /dev/null
 
 # Replace the loopback address with the cluster hostname

.github/test-infra/aws/rke2/scripts/helmchart-template.yaml

@@ -0,0 +1,107 @@
+# Copyright 2025 Defense Unicorns
+# SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial
+
+apiVersion: helm.cattle.io/v1
+kind: HelmChart
+metadata:
+  name: aws-cloud-controller-manager
+  namespace: kube-system
+spec:
+  chart: aws-cloud-controller-manager
+  repo: https://kubernetes.github.io/cloud-provider-aws
+  # renovate: datasource=helm depName=aws-cloud-controller-manager versioning=helm registryUrl=https://kubernetes.github.io/cloud-provider-aws
+  version: 0.0.8
+  targetNamespace: kube-system
+  bootstrap: true
+  valuesContent: |-
+    nodeSelector:
+      node-role.kubernetes.io/control-plane: "true"
+    hostNetworking: true
+    args:
+      - --configure-cloud-routes=false
+      - --v=2
+      - --cloud-provider=aws
+---
+# aws lb controller helm values: https://github.com/kubernetes-sigs/aws-load-balancer-controller/tree/main/helm/aws-load-balancer-controller#configuration
+apiVersion: helm.cattle.io/v1
+kind: HelmChart
+metadata:
+  name: aws-load-balancer-controller
+  namespace: kube-system
+spec:
+  chart: aws-load-balancer-controller
+  repo: https://aws.github.io/eks-charts
+  # renovate: datasource=helm depName=aws-load-balancer-controller versioning=helm registryUrl=https://aws.github.io/eks-charts
+  version: 1.12.0
+  targetNamespace: kube-system
+  valuesContent: |-
+    clusterName: ${CLUSTER_NAME}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: coredns-custom
+  namespace: kube-system
+data:
+  uds.override: |
+    rewrite stop {
+      name regex (.*\.admin\.uds\.dev) admin-ingressgateway.istio-admin-gateway.svc.cluster.local answer auto
+    }
+    rewrite stop {
+      name regex (.*\.uds\.dev) tenant-ingressgateway.istio-tenant-gateway.svc.cluster.local answer auto
+    }
+---
+apiVersion: helm.cattle.io/v1
+kind: HelmChartConfig
+metadata:
+  name: rke2-coredns
+  namespace: kube-system
+spec:
+  valuesContent: |-
+    extraVolumes:
+      - name: custom-config-volume
+        configMap:
+          name: coredns-custom
+          optional: true
+    extraVolumeMounts:
+      - name: custom-config-volume
+        mountPath: /etc/coredns/custom
+        readOnly: true
+    # Below we take the default kubernetes configmap for coredns and add an import statement for our custom configmap
+    # Ref: https://github.com/rancher/rke2-charts/blob/8078e4184e5b1730e518344aaa170a5e49e29766/charts/rke2-coredns/rke2-coredns/1.39.101/values.yaml#L104
+    servers:
+    - zones:
+      - zone: .
+      port: 53
+      # -- expose the service on a different port
+      # servicePort: 5353
+      # If serviceType is nodePort you can specify nodePort here
+      # nodePort: 30053
+      # hostPort: 53
+      plugins:
+      - name: errors
+      # Serves a /health endpoint on :8080, required for livenessProbe
+      - name: health
+        configBlock: |-
+          lameduck 5s
+      # Serves a /ready endpoint on :8181, required for readinessProbe
+      - name: ready
+      # Required to query kubernetes API for data
+      - name: kubernetes
+        parameters: cluster.local in-addr.arpa ip6.arpa
+        configBlock: |-
+          pods insecure
+          fallthrough in-addr.arpa ip6.arpa
+          ttl 30
+      # Serves a /metrics endpoint on :9153, required for serviceMonitor
+      - name: prometheus
+        parameters: 0.0.0.0:9153
+      - name: forward
+        parameters: . /etc/resolv.conf
+      - name: cache
+        parameters: 30
+      - name: loop
+      - name: reload
+      - name: loadbalance
+      - name: import
+        parameters: /etc/coredns/custom/*.override

.github/test-infra/aws/rke2/scripts/user_data.sh

@@ -2,79 +2,45 @@
 # Copyright 2024 Defense Unicorns
 # SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial
 
-
-
 info() {
     echo "[INFO] " "$@"
 }
 
 export CCM="${ccm}"
 export CCM_EXTERNAL="${ccm_external}"
+export CLUSTER_NAME="${cluster_name}"
 
 ###############################
 ### pre userdata
 ###############################
 pre_userdata() {
 info "Beginning user defined pre userdata"
-
-# add aws cloud controller
-info "Adding AWS cloud provider manifest."
+info "Create HelmChart Resources."
 mkdir -p /var/lib/rancher/rke2/server/manifests
-cat > /var/lib/rancher/rke2/server/manifests/00-aws-ccm.yaml << EOM
-apiVersion: helm.cattle.io/v1
-kind: HelmChart
-metadata:
-  name: aws-cloud-controller-manager
-  namespace: kube-system
-spec:
-  chart: aws-cloud-controller-manager
-  repo: https://kubernetes.github.io/cloud-provider-aws
-  version: 0.0.8
-  targetNamespace: kube-system
-  bootstrap: true
-  valuesContent: |-
-    nodeSelector:
-      node-role.kubernetes.io/control-plane: "true"
-    hostNetworking: true
-    args:
-      - --configure-cloud-routes=false
-      - --v=2
-      - --cloud-provider=aws
+cat > helmchart-template.yaml << EOM
+${helm_chart_template}
 EOM
 
-#longhorn helm values: https://github.com/longhorn/longhorn/tree/master/chart
-cat > /var/lib/rancher/rke2/server/manifests/01-longhorn.yaml << EOM
-apiVersion: helm.cattle.io/v1
-kind: HelmChart
-metadata:
-  name: longhorn
-  namespace: kube-system
-spec:
-  chart: longhorn
-  repo: https://charts.longhorn.io
-  version: 1.7.1
-  targetNamespace: kube-system 
-EOM
-
-#metallb helm values: https://github.com/metallb/metallb/tree/main/charts/metallb
-cat > /var/lib/rancher/rke2/server/manifests/02-metallb.yaml << EOM
-apiVersion: helm.cattle.io/v1
-kind: HelmChart
-metadata:
-  name: metallb
-  namespace: kube-system
-spec:
-  chart: metallb
-  repo: https://metallb.github.io/metallb
-  version: 0.14.8
-  targetNamespace: kube-system
-EOM
+envsubst < helmchart-template.yaml > /var/lib/rancher/rke2/server/manifests/00-helmcharts.yaml
+# We install longhorn from a template to avoid install issues with the HelmController
+# <!-- renovate: datasource=helm depName=longhorn versioning=helm registryUrl=https://charts.longhorn.io -->
+LONGHORN_VERSION=1.8.1
+HELM_LATEST=$(curl -L --silent --show-error --fail "https://get.helm.sh/helm-latest-version" 2>&1 || true)
+curl https://get.helm.sh/helm-$HELM_LATEST-linux-amd64.tar.gz --output helm.tar.gz
+tar -xvf ./helm.tar.gz && rm -rf ./helm.tar.gz
+chmod +x ./linux-amd64/helm
+./linux-amd64/helm repo add longhorn https://charts.longhorn.io
+./linux-amd64/helm repo update
+./linux-amd64/helm template longhorn longhorn/longhorn --version $LONGHORN_VERSION --set defaultSettings.deletingConfirmationFlag=true --set longhornUI.replicas=0 --set namespaceOverride=kube-system --no-hooks > /var/lib/rancher/rke2/server/manifests/01-longhorn.yaml
+rm -rf ./linux-amd64
 
 info "Installing awscli"
 yum install -y unzip jq || apt-get -y install unzip jq
 curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
 unzip awscliv2.zip
 sudo ./aws/install
+curl -L https://github.com/mikefarah/yq/releases/download/v4.40.4/yq_linux_amd64 -o yq
+chmod +x yq
 
 echo "Getting OIDC keypair"
 sudo mkdir /irsa
@@ -84,19 +50,18 @@ aws secretsmanager get-secret-value --secret-id ${secret_prefix}-oidc-public-key
 chcon -t svirt_sandbox_file_t /irsa/*
 
 info "Setting up RKE2 config file"
-curl -L https://github.com/mikefarah/yq/releases/download/v4.40.4/yq_linux_amd64 -o yq
-chmod +x yq
 ./yq -i '.cloud-provider-name += "external"' /etc/rancher/rke2/config.yaml
 ./yq -i '.disable-cloud-controller += "true"' /etc/rancher/rke2/config.yaml
 ./yq -i '.kube-apiserver-arg += "service-account-key-file=/irsa/signer.key.pub"' /etc/rancher/rke2/config.yaml
-./yq -i '.kube-apiserver-arg += "service-account-key-file=/irsa/signer.key.pub"' /etc/rancher/rke2/config.yaml
 ./yq -i '.kube-apiserver-arg += "service-account-signing-key-file=/irsa/signer.key"' /etc/rancher/rke2/config.yaml
 ./yq -i '.kube-apiserver-arg += "api-audiences=kubernetes.svc.default"' /etc/rancher/rke2/config.yaml
 ./yq -i '.kube-apiserver-arg += "service-account-issuer=https://${BUCKET_REGIONAL_DOMAIN_NAME}"' /etc/rancher/rke2/config.yaml
 ./yq -i '.kube-apiserver-arg += "audit-log-path=/var/log/kubernetes/audit/audit.log"' /etc/rancher/rke2/config.yaml
+#Fix for metrics server scraping of kubernetes api server components
+./yq -i '.kube-controller-manager-arg[2] = "bind-address=0.0.0.0"' /etc/rancher/rke2/config.yaml
+./yq -i '.kube-scheduler-arg += "bind-address=0.0.0.0"' /etc/rancher/rke2/config.yaml
+./yq -i '.etcd-arg += "listen-metrics-urls=http://0.0.0.0:2381"|.etcd-arg style="double"' /etc/rancher/rke2/config.yaml
 rm -rf ./yq
-
-
 }
 
 pre_userdata

.github/workflows/test-aks.yaml

@@ -88,12 +88,19 @@ jobs:
       - name: Create IAC
         run: uds run -f tasks/iac.yaml apply-tofu --no-progress --set K8S_DISTRO=aks --set CLOUD=azure
 
+      - name: Configure Cluster DNS
+        run: uds run -f tasks/utils.yaml aks-coredns-setup --no-progress
+
       - name: Deploy Core Bundle
         env:
           UDS_CONFIG: .github/bundles/aks/uds-config.yaml
         run: uds deploy .github/bundles/aks/uds-bundle-uds-core-aks-nightly-*.tar.zst --confirm
         timeout-minutes: 30
 
+      - name: Test UDS Core
+        run: uds run -f tasks/test.yaml uds-core-non-k3d --set EXCLUDED_PACKAGES="metrics-server"
+        continue-on-error: true
+
       - name: Debug Output
         if: ${{ always() }}
         uses: ./.github/actions/debug-output

.github/workflows/test-eks.yaml

@@ -92,12 +92,19 @@ jobs:
         run: uds run -f tasks/iac.yaml create-iac --no-progress --set K8S_DISTRO=eks --set CLOUD=aws
         timeout-minutes: 20
 
+      - name: Configure Cluster DNS
+        run: uds run -f tasks/utils.yaml eks-coredns-setup --no-progress
+
       - name: Deploy Core Bundle
         env:
           UDS_CONFIG: .github/bundles/eks/uds-config.yaml
         run: uds deploy .github/bundles/eks/uds-bundle-uds-core-eks-nightly-*.tar.zst --confirm
         timeout-minutes: 30
 
+      - name: Test UDS Core
+        run: uds run -f tasks/test.yaml uds-core-non-k3d --set EXCLUDED_PACKAGES="metrics-server"
+        continue-on-error: true
+
       - name: Debug Output
         if: ${{ always() }}
         uses: ./.github/actions/debug-output

.github/workflows/test-rke2.yaml

@@ -101,6 +101,10 @@ jobs:
         run: uds deploy .github/bundles/rke2/uds-bundle-uds-core-rke2-nightly-*.tar.zst --confirm
         timeout-minutes: 30
 
+      - name: Test UDS Core
+        run: uds run -f tasks/test.yaml uds-core-non-k3d
+        continue-on-error: true
+
       - name: Debug Output
         if: ${{ always() }}
         uses: ./.github/actions/debug-output

src/istio/tasks.yaml

@@ -6,7 +6,7 @@ tasks:
     inputs:
       validate_passthrough:
         description: Whether to validate the passthrough gateway
-        default: "true"
+
     actions:
       - description: Validate the Istio Admin Gateway
         wait: