chore: add nightly testing for AKS (#908)

## Description
Adds nightly testing for uds-core on AKS

Callouts:
- Removes `nightly-testing.yaml` in favor of a single workflow for each
distribution
- Deploys storage account and containers for velero and loki and
configures uds-core to use them
- Deploys postgresql database and configures grafana to use it for HA
configuration
- adds `uds-config.tf` file and writes `uds-config.yaml` using terraform
`local_sensitive_file` instead of `tf output xyz >> uds-config.yaml`
pattern used in the past

## Related Issues
Fixes:
  - #727
  - #856


## Type of change

- [ ] Bug fix (non-breaking change which fixes an issue)
- [x] New feature (non-breaking change which adds functionality)
- [ ] Other (security config, docs update, etc)

## Checklist before merging

- [x] Test, docs, adr added or updated as needed
- [ ] [Contributor
Guide](https://github.com/defenseunicorns/uds-template-capability/blob/main/CONTRIBUTING.md)
followed

---------

Co-authored-by: Micah Nagel <[email protected]>

.github/actions/debug-output/action.yaml

@@ -22,6 +22,6 @@ runs:
         uds zarf tools kubectl get events -A --sort-by='.lastTimestamp' | tee /tmp/debug-k-get-events.log || true
         echo "::endgroup::"
         echo "::group::kubectl describe nodes"
-        uds zarf tools kubectl describe nodes k3d-uds-server-0 | tee /tmp/debug-k-describe-node.log || true
+        uds zarf tools kubectl describe nodes | tee /tmp/debug-k-describe-node.log || true
         echo "::endgroup::"
       shell: bash

.github/bundles/aks/uds-bundle.yaml

@@ -0,0 +1,137 @@
+# Copyright 2024 Defense Unicorns
+# SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial
+
+kind: UDSBundle
+metadata:
+  name: uds-core-aks-nightly
+  description: A UDS bundle for deploying UDS Core on AKS
+  version: "0.29.0"
+
+packages:
+  - name: init
+    repository: ghcr.io/zarf-dev/packages/init
+    ref: v0.41.0
+
+  - name: core
+    path: ../../../build
+    # x-release-please-start-version
+    ref: 0.29.0
+    # x-release-please-end
+    overrides:
+      istio-admin-gateway:
+        gateway:
+          values:
+            - path: service.annotations
+              value:
+                service.beta.kubernetes.io/azure-load-balancer-internal: "false"
+                service.beta.kubernetes.io/azure-load-balancer-sku: "Standard"
+                service.beta.kubernetes.io/azure-load-balancer-resource-group: "${NODE_RESOURCE_GROUP_NAME}"
+
+      istio-tenant-gateway:
+        gateway:
+          values:
+            - path: service.annotations
+              value:
+                service.beta.kubernetes.io/azure-load-balancer-internal: "false"
+                service.beta.kubernetes.io/azure-load-balancer-sku: "Standard"
+                service.beta.kubernetes.io/azure-load-balancer-resource-group: "${NODE_RESOURCE_GROUP_NAME}"
+      loki:
+        loki:
+          variables:
+            - name: AZURE_LOKI_STORAGE_ACCOUNT
+              description: "Name of the Storage Account to use for storing logs"
+              path: "loki.storage_config.azure.account_name"
+            - name: AZURE_LOKI_STORAGE_ACCOUNT_ACCESS_KEY
+              description: "Primary access Key for the Storage Account"
+              path: "loki.storage_config.azure.account_key"
+            - name: AZURE_LOKI_STORAGE_ACCOUNT_CONTAINER
+              description: "The destination container in the Storage Account where logs will be saved"
+              path: "loki.storage_config.azure.container_name"
+          values:
+            - path: loki.storage.type
+              value: "azure"
+
+      kube-prometheus-stack:
+        kube-prometheus-stack:
+          values:
+            - path: kube-state-metrics
+              value:
+                resources:
+                  limits:
+                    memory: 512Mi
+      grafana:
+        grafana:
+          variables:
+            - name: GRAFANA_HA
+              description: Enable HA Grafana
+              path: autoscaling.enabled
+        uds-grafana-config:
+          variables:
+            - name: GRAFANA_PG_HOST
+              description: Grafana postgresql host
+              path: postgresql.host
+            - name: GRAFANA_PG_PORT
+              description: Grafana postgresql port
+              path: postgresql.port
+            - name: GRAFANA_PG_DATABASE
+              description: Grafana postgresql database
+              path: postgresql.database
+            - name: GRAFANA_PG_PASSWORD
+              description: Grafana postgresql password
+              path: postgresql.password
+            - name: GRAFANA_PG_USER
+              description: Grafana postgresql username
+              path: postgresql.user
+
+      neuvector:
+        core:
+          values:
+            - path: runtimePath
+              value: /run/containerd/containerd.sock
+            - path: enforcer.tolerations
+              value:
+                - effect: NoSchedule
+                  key: node-role.kubernetes.io/master
+                - effect: NoSchedule
+                  key: node-role.kubernetes.io/control-plane
+                - effect: NoSchedule
+                  key: dedicated
+                  operator: Exists
+      velero:
+        velero:
+          variables:
+            - name: AZURE_VELERO_STORAGE_ACCOUNT
+              description: "Name of the Storage Account to use for storing backups"
+              path: "configuration.backupStorageLocation[0].config.storageAccount"
+            - name: AZURE_VELERO_STORAGE_ACCOUNT_ACCESS_KEY
+              description: "Primary access Key for the Storage Account"
+              path: "configuration.backupStorageLocation[0].config.storageAccountKeyEnvVar"
+            - name: AZURE_VELERO_STORAGE_ACCOUNT_CONTAINER
+              description: "The destination container in the Storage Account where backups will be saved"
+              path: "configuration.backupStorageLocation[0].bucket"
+            - name: AZURE_RESOURCE_GROUP
+              description: "The name of the resource group that the Storage Account is in"
+              path: "configuration.backupStorageLocation[0].config.resourceGroup"
+            - name: AZURE_SUBSCRIPTION_ID
+              description: "The resource ID of the Azure Subscription that is being used"
+              path: "configuration.backupStorageLocation[0].config.subscriptionId"
+            - name: VELERO_CLIENT_SECRET_ENV_VAR
+              description: "Name of the env variable that velero will use to read Azure config"
+              path: "configuration.backupStorageLocation[0].config.storageAccountKeyEnvVar"
+              default: "AZURE_STORAGE_ACCOUNT_ACCESS_KEY"
+            - name: VELERO_BACKUP_STORAGE_CONFIG_NAME
+              description: "Name of the Backup Storage Location"
+              path: "configuration.backupStorageLocation[0].name"
+              default: "default"
+            - name: VELERO_STORAGE_PROVIDER
+              description: "Type of storage provider that will be used"
+              path: "configuration.backupStorageLocation[0].provider"
+              default: "azure"
+          values:
+            - path: credentials
+              value:
+                useSecret: true
+                secretContents:
+                  cloud: |
+                    AZURE_STORAGE_ACCOUNT_ACCESS_KEY=${AZURE_VELERO_STORAGE_ACCOUNT_ACCESS_KEY}
+                    AZURE_CLOUD_NAME=AzurePublicCloud

.github/bundles/aks/uds-config.yaml

@@ -0,0 +1,14 @@
+# Copyright 2024 Defense Unicorns
+# SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial
+
+# Overwritten in ci by uds-config.tf in test-infra/azure/aks
+variables:
+  core:
+    azure_loki_storage_account: ${ZARF_VAR_AZURE_LOKI_STORAGE_ACCOUNT}
+    azure_loki_storage_account_access_key: ${ZARF_VAR_AZURE_LOKI_STORAGE_ACCOUNT_ACCESS_KEY}
+    azure_loki_storage_account_container: ${ZARF_VAR_AZURE_LOKI_STORAGE_ACCOUNT_CONTAINER}
+    azure_velero_storage_account: ${ZARF_VAR_AZURE_VELERO_STORAGE_ACCOUNT}
+    azure_velero_storage_account_acces_key: ${ZARF_VAR_AZURE_VELERO_STORAGE_ACCOUNT_ACCESS_KEY}
+    azure_velero_storage_account_container: ${ZARF_VAR_AZURE_VELERO_STORAGE_ACCOUNT_CONTAINER}
+    azure_subscription_id: ${ZARF_VAR_AZURE_SUBSCRIPTION_ID}
+    azure_resource_group: ${ZARF_VAR_AZURE_RESOURCE_GROUP}

.github/test-infra/aws/eks/main.tf

@@ -1,7 +1,6 @@
 # Copyright 2024 Defense Unicorns
 # SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial
 
-
 resource "random_id" "default" {
   byte_length = 2
 }

.github/test-infra/aws/eks/uds-config.tf

@@ -0,0 +1,33 @@
+# Copyright 2024 Defense Unicorns
+# SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial
+
+resource "local_sensitive_file" "uds_config" {
+  filename = "../../../bundles/eks/uds-config.yaml"
+  content = yamlencode({
+    "options" : {
+      "architecture" : "amd64"
+    },
+    "variables" : {
+      "core" : {
+        "loki_chunks_bucket" : module.S3["loki"].bucket_name
+        "loki_ruler_bucket" : module.S3["loki"].bucket_name,
+        "loki_admin_bucket" : module.S3["loki"].bucket_name,
+        "loki_s3_region" : data.aws_region.current.name,
+        "loki_irsa_role_arn" : module.irsa["loki"].role_arn,
+        "velero_use_secret" : false,
+        "velero_irsa_role_arn" : module.irsa["velero"].role_arn,
+        "velero_bucket" : module.S3["velero"].bucket_name,
+        "velero_bucket_region" : data.aws_region.current.name,
+        "velero_bucket_provider_url" : "",
+        "velero_bucket_credential_name" : "",
+        "velero_bucket_credential_key" : "",
+        "grafana_ha" : true,
+        "grafana_pg_host" : element(split(":", module.db.db_instance_endpoint), 0),
+        "grafana_pg_port" : var.db_port,
+        "grafana_pg_database" : var.db_name,
+        "grafana_pg_password" : random_password.db_password.result,
+        "grafana_pg_user" : var.username
+      }
+    }
+  })
+}

.github/test-infra/aws/rke2/irsa.tf

@@ -50,7 +50,6 @@ module "oidc_bucket" {
   restrict_public_buckets  = false
 }
 
-
 # OIDC file creation
 resource "local_file" "oidc_config" {
   content  = <<EOF

.github/test-infra/aws/rke2/scripts/get-kubeconfig.sh

@@ -32,7 +32,9 @@ scp -o StrictHostKeyChecking=no -i key.pem ${node_user}@${bootstrap_ip}:/home/${
 
 # Replace the loopback address with the cluster hostname
 sed -i "s/127.0.0.1/${bootstrap_ip}/g" ./rke2-config > /dev/null
-export KUBECONFIG=$(pwd)/rke2-config
+mkdir -p /home/runner/.kube
+mv ./rke2-config /home/runner/.kube/config
+#export KUBECONFIG=$(pwd)/rke2-config
 
 # find existing host record in the host file and save the line numbers
 matches_in_hosts="$(grep -n $cluster_hostname /etc/hosts | cut -f1 -d:)"

.github/test-infra/aws/rke2/uds-config.tf

@@ -0,0 +1,33 @@
+# Copyright 2024 Defense Unicorns
+# SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial
+
+resource "local_sensitive_file" "uds_config" {
+  filename = "../../../bundles/rke2/uds-config.yaml"
+  content = yamlencode({
+    "options" : {
+      "architecture" : "amd64"
+    },
+    "variables" : {
+      "core" : {
+        "loki_chunks_bucket" : module.storage.s3_buckets["loki"].bucket_name
+        "loki_ruler_bucket" : module.storage.s3_buckets["loki"].bucket_name,
+        "loki_admin_bucket" : module.storage.s3_buckets["loki"].bucket_name,
+        "loki_s3_region" : data.aws_region.current.name,
+        "loki_irsa_role_arn" : module.storage.irsa["loki"].bucket_role.arn
+        "velero_use_secret" : false,
+        "velero_irsa_role_arn" : module.storage.irsa["velero"].bucket_role.arn,
+        "velero_bucket" : module.storage.s3_buckets["velero"].bucket_name,
+        "velero_bucket_region" : data.aws_region.current.name,
+        "velero_bucket_provider_url" : ""
+        "velero_bucket_credential_name" : "",
+        "velero_bucket_credential_key" : "",
+        "grafana_ha" : false,
+        "grafana_pg_host" : "\"\"",
+        "grafana_pg_port" : "\"\"",
+        "grafana_pg_database" : "\"\"",
+        "grafana_pg_password" : "\"\"",
+        "grafana_pg_user" : "\"\"",
+      }
+    }
+  })
+}

.github/test-infra/azure/aks/main.tf

@@ -0,0 +1,108 @@
+# Copyright 2024 Defense Unicorns
+# SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial
+
+
+data "azurerm_client_config" "current" {}
+
+locals {
+  cluster_name = "${var.cluster_name}-${random_string.name.result}"
+}
+
+resource "random_string" "name" {
+  length  = 4
+  special = false
+  upper   = false
+  numeric = false
+}
+
+## resource group that cluster will be created in
+resource "azurerm_resource_group" "this" {
+  name     = "${var.resource_group_name}-${random_string.name.result}"
+  location = var.location
+}
+
+resource "azurerm_role_assignment" "cluster_admin" {
+  scope                = azurerm_kubernetes_cluster.aks_cluster.id
+  role_definition_name = "Azure Kubernetes Service RBAC Cluster Admin"
+  principal_id         = data.azurerm_client_config.current.object_id
+}
+
+resource "azurerm_role_assignment" "aks_network_role" {
+  principal_id         = azurerm_kubernetes_cluster.aks_cluster.identity[0].principal_id
+  role_definition_name = "Network Contributor"
+  scope                = azurerm_resource_group.this.id
+}
+
+### CSI Driver identity. Required if workload_identity_enabled is true
+resource "azurerm_user_assigned_identity" "workload_identity" {
+  count               = var.workload_identity_enabled ? 1 : 0
+  location            = var.location
+  name                = "${local.cluster_name}-workload-identity"
+  resource_group_name = azurerm_resource_group.this.name
+}
+
+resource "azurerm_kubernetes_cluster" "aks_cluster" {
+  name                      = local.cluster_name
+  location                  = var.location
+  resource_group_name       = azurerm_resource_group.this.name
+  kubernetes_version        = var.kubernetes_version
+  dns_prefix                = var.dns_prefix
+  sku_tier                  = var.sku_tier
+  workload_identity_enabled = var.workload_identity_enabled
+  oidc_issuer_enabled       = var.oidc_issuer_enabled
+
+  default_node_pool {
+    name                 = var.default_node_pool_name
+    auto_scaling_enabled = var.enable_autoscaling
+    vnet_subnet_id       = azurerm_subnet.cluster_node_subnet.id
+    max_count            = var.autoscaling_max_node_count
+    min_count            = var.autoscaling_min_node_count
+    vm_size              = var.default_node_pool_vm_size
+    zones                = var.default_node_pool_availability_zones
+    node_labels          = var.default_node_pool_node_labels
+    max_pods             = var.default_node_pool_max_pods
+    node_count           = var.default_node_pool_node_count
+    os_disk_type         = var.default_node_pool_os_disk_type
+    tags                 = var.tags
+  }
+
+  identity {
+    type = "SystemAssigned"
+  }
+
+  azure_active_directory_role_based_access_control {
+    azure_rbac_enabled     = true
+    admin_group_object_ids = []
+  }
+
+  dynamic "key_vault_secrets_provider" {
+    for_each = var.enable_key_vault_csi_driver ? { "enabled" = true } : {}
+    content {
+      secret_rotation_enabled = key_vault_secrets_provider.value
+    }
+  }
+
+  storage_profile {
+    blob_driver_enabled = false
+    file_driver_enabled = true
+  }
+
+  network_profile {
+    dns_service_ip = var.network_dns_service_ip
+    network_plugin = var.network_plugin
+    network_policy = var.network_policy
+    outbound_type  = var.outbound_type
+    service_cidr   = var.network_service_cidr
+  }
+
+  lifecycle {
+    ignore_changes = [
+      kubernetes_version,
+      tags
+    ]
+  }
+
+  depends_on = [
+    azurerm_resource_group.this
+  ]
+}