diff --git a/.github/bundles/rke2/uds-bundle.yaml b/.github/bundles/rke2/uds-bundle.yaml index 734d555a30..1eb4e6827a 100644 --- a/.github/bundles/rke2/uds-bundle.yaml +++ b/.github/bundles/rke2/uds-bundle.yaml @@ -43,6 +43,20 @@ packages: optionalComponents: - metrics-server overrides: + istio-admin-gateway: + gateway: + values: + - path: service.annotations + value: + service.beta.kubernetes.io/aws-load-balancer-scheme: "internet-facing" + service.beta.kubernetes.io/aws-load-balancer-target-node-labels: "kubernetes.io/os=linux" + istio-tenant-gateway: + gateway: + values: + - path: service.annotations + value: + service.beta.kubernetes.io/aws-load-balancer-scheme: "internet-facing" + service.beta.kubernetes.io/aws-load-balancer-target-node-labels: "kubernetes.io/os=linux" velero: velero: variables: diff --git a/.github/test-infra/aws/rke2/data.tf b/.github/test-infra/aws/rke2/data.tf index a6bae125f8..d57af5d93f 100644 --- a/.github/test-infra/aws/rke2/data.tf +++ b/.github/test-infra/aws/rke2/data.tf @@ -10,7 +10,7 @@ data "aws_vpc" "vpc" { data "aws_subnet" "rke2_ci_subnet" { vpc_id = data.aws_vpc.vpc.id - availability_zone = "${var.region}c" + availability_zone = "${var.region}a" filter { name = "tag:Name" diff --git a/.github/test-infra/aws/rke2/iam.tf b/.github/test-infra/aws/rke2/iam.tf index 09d90a947b..7bceb5fbec 100644 --- a/.github/test-infra/aws/rke2/iam.tf +++ b/.github/test-infra/aws/rke2/iam.tf @@ -77,6 +77,19 @@ data "aws_iam_policy_document" "aws_ccm" { } } +data "local_file" "helm_template" { + filename = "./scripts/helmchart-template.yaml" +} + +data "http" "aws-lb-controller-iam" { + url = "https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/v2.12.0/docs/install/iam_policy_us-gov.json" +} +resource "aws_iam_role_policy" "aws-lb-controller" { + name = "${local.cluster_name}-lb-controller" + role = aws_iam_role.rke2_server.id + policy = data.http.aws-lb-controller-iam.response_body +} + resource "aws_iam_role_policy" "s3_token" { name = "${local.cluster_name}-server-token" role = aws_iam_role.rke2_server.id diff --git a/.github/test-infra/aws/rke2/main.tf b/.github/test-infra/aws/rke2/main.tf index 9fb44ef06f..8152b5c3bd 100644 --- a/.github/test-infra/aws/rke2/main.tf +++ b/.github/test-infra/aws/rke2/main.tf @@ -26,6 +26,8 @@ locals { ccm_external = true, token_bucket = module.statestore.bucket, token_object = module.statestore.token_object + cluster_name = local.tags.cluster_name + helm_chart_template = file("./scripts/helmchart-template.yaml") } } @@ -95,7 +97,7 @@ resource "aws_instance" "rke2_ci_control_plane_node" { associate_public_ip_address = true root_block_device { - volume_size = 100 + volume_size = 250 } tags = merge(local.tags, { "kubernetes.io/cluster/${local.cluster_name}" = "owned" }) @@ -107,15 +109,16 @@ resource "aws_instance" "rke2_ci_agent_node" { ami = data.aws_ami.rhel_rke2.image_id instance_type = var.agent_instance_type key_name = aws_key_pair.control_plane_key_pair.key_name - user_data = templatefile("${path.module}/scripts/user_data.sh", merge(local.userdata, { BOOTSTRAP_IP = aws_instance.rke2_ci_bootstrap_node.private_ip })) + user_data = templatefile("${path.module}/scripts/user_data.sh", merge(local.userdata, { BOOTSTRAP_IP = aws_instance.rke2_ci_bootstrap_node.private_ip, AGENT_NODE = true })) subnet_id = data.aws_subnet.rke2_ci_subnet.id user_data_replace_on_change = true iam_instance_profile = aws_iam_instance_profile.rke2_server.name vpc_security_group_ids = [aws_security_group.rke2_ci_node_sg.id] associate_public_ip_address = true + availability_zone = "${var.region}a" root_block_device { - volume_size = 100 + volume_size = 250 } tags = merge(local.tags, { "kubernetes.io/cluster/${local.cluster_name}" = "owned" }) diff --git a/.github/test-infra/aws/rke2/scripts/get-kubeconfig.sh b/.github/test-infra/aws/rke2/scripts/get-kubeconfig.sh old mode 100644 new mode 100755 index 472a816695..967a44af00 --- a/.github/test-infra/aws/rke2/scripts/get-kubeconfig.sh +++ b/.github/test-infra/aws/rke2/scripts/get-kubeconfig.sh @@ -27,7 +27,7 @@ done mkdir -p ~/.kube # Copy kubectl from cluster node -ssh -o StrictHostKeyChecking=no -i key.pem ${node_user}@${bootstrap_ip} "mkdir -p /home/${node_user}/.kube && sudo cp /etc/rancher/rke2/rke2.yaml /home/${node_user}/.kube/config && sudo chown ${node_user} /home/${node_user}/.kube/config" > /dev/null +ssh -o StrictHostKeyChecking=no -i key.pem ${node_user}@${bootstrap_ip} "mkdir -p /home/${node_user}/.kube && sudo cp /etc/rancher/rke2/rke2.yaml /home/${node_user}/.kube/config && sudo chown ${node_user} /home/${node_user}/.kube/config" > /dev/null scp -o StrictHostKeyChecking=no -i key.pem ${node_user}@${bootstrap_ip}:/home/${node_user}/.kube/config ./rke2-config > /dev/null # Replace the loopback address with the cluster hostname diff --git a/.github/test-infra/aws/rke2/scripts/helmchart-template.yaml b/.github/test-infra/aws/rke2/scripts/helmchart-template.yaml new file mode 100644 index 0000000000..801bf0bb2f --- /dev/null +++ b/.github/test-infra/aws/rke2/scripts/helmchart-template.yaml @@ -0,0 +1,107 @@ +# Copyright 2025 Defense Unicorns +# SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial + +apiVersion: helm.cattle.io/v1 +kind: HelmChart +metadata: + name: aws-cloud-controller-manager + namespace: kube-system +spec: + chart: aws-cloud-controller-manager + repo: https://kubernetes.github.io/cloud-provider-aws + # renovate: datasource=helm depName=aws-cloud-controller-manager versioning=helm registryUrl=https://kubernetes.github.io/cloud-provider-aws + version: 0.0.8 + targetNamespace: kube-system + bootstrap: true + valuesContent: |- + nodeSelector: + node-role.kubernetes.io/control-plane: "true" + hostNetworking: true + args: + - --configure-cloud-routes=false + - --v=2 + - --cloud-provider=aws +--- +# aws lb controller helm values: https://github.com/kubernetes-sigs/aws-load-balancer-controller/tree/main/helm/aws-load-balancer-controller#configuration +apiVersion: helm.cattle.io/v1 +kind: HelmChart +metadata: + name: aws-load-balancer-controller + namespace: kube-system +spec: + chart: aws-load-balancer-controller + repo: https://aws.github.io/eks-charts + # renovate: datasource=helm depName=aws-load-balancer-controller versioning=helm registryUrl=https://aws.github.io/eks-charts + version: 1.12.0 + targetNamespace: kube-system + valuesContent: |- + clusterName: ${CLUSTER_NAME} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: coredns-custom + namespace: kube-system +data: + uds.override: | + rewrite stop { + name regex (.*\.admin\.uds\.dev) admin-ingressgateway.istio-admin-gateway.svc.cluster.local answer auto + } + rewrite stop { + name regex (.*\.uds\.dev) tenant-ingressgateway.istio-tenant-gateway.svc.cluster.local answer auto + } +--- +apiVersion: helm.cattle.io/v1 +kind: HelmChartConfig +metadata: + name: rke2-coredns + namespace: kube-system +spec: + valuesContent: |- + extraVolumes: + - name: custom-config-volume + configMap: + name: coredns-custom + optional: true + extraVolumeMounts: + - name: custom-config-volume + mountPath: /etc/coredns/custom + readOnly: true + # Below we take the default kubernetes configmap for coredns and add an import statement for our custom configmap + # Ref: https://github.com/rancher/rke2-charts/blob/8078e4184e5b1730e518344aaa170a5e49e29766/charts/rke2-coredns/rke2-coredns/1.39.101/values.yaml#L104 + servers: + - zones: + - zone: . + port: 53 + # -- expose the service on a different port + # servicePort: 5353 + # If serviceType is nodePort you can specify nodePort here + # nodePort: 30053 + # hostPort: 53 + plugins: + - name: errors + # Serves a /health endpoint on :8080, required for livenessProbe + - name: health + configBlock: |- + lameduck 5s + # Serves a /ready endpoint on :8181, required for readinessProbe + - name: ready + # Required to query kubernetes API for data + - name: kubernetes + parameters: cluster.local in-addr.arpa ip6.arpa + configBlock: |- + pods insecure + fallthrough in-addr.arpa ip6.arpa + ttl 30 + # Serves a /metrics endpoint on :9153, required for serviceMonitor + - name: prometheus + parameters: 0.0.0.0:9153 + - name: forward + parameters: . /etc/resolv.conf + - name: cache + parameters: 30 + - name: loop + - name: reload + - name: loadbalance + - name: import + parameters: /etc/coredns/custom/*.override diff --git a/.github/test-infra/aws/rke2/scripts/user_data.sh b/.github/test-infra/aws/rke2/scripts/user_data.sh index 82b9b7cc6a..92ee131a59 100644 --- a/.github/test-infra/aws/rke2/scripts/user_data.sh +++ b/.github/test-infra/aws/rke2/scripts/user_data.sh @@ -2,79 +2,45 @@ # Copyright 2024 Defense Unicorns # SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial - - info() { echo "[INFO] " "$@" } export CCM="${ccm}" export CCM_EXTERNAL="${ccm_external}" +export CLUSTER_NAME="${cluster_name}" ############################### ### pre userdata ############################### pre_userdata() { info "Beginning user defined pre userdata" - -# add aws cloud controller -info "Adding AWS cloud provider manifest." +info "Create HelmChart Resources." mkdir -p /var/lib/rancher/rke2/server/manifests -cat > /var/lib/rancher/rke2/server/manifests/00-aws-ccm.yaml << EOM -apiVersion: helm.cattle.io/v1 -kind: HelmChart -metadata: - name: aws-cloud-controller-manager - namespace: kube-system -spec: - chart: aws-cloud-controller-manager - repo: https://kubernetes.github.io/cloud-provider-aws - version: 0.0.8 - targetNamespace: kube-system - bootstrap: true - valuesContent: |- - nodeSelector: - node-role.kubernetes.io/control-plane: "true" - hostNetworking: true - args: - - --configure-cloud-routes=false - - --v=2 - - --cloud-provider=aws +cat > helmchart-template.yaml << EOM +${helm_chart_template} EOM -#longhorn helm values: https://github.com/longhorn/longhorn/tree/master/chart -cat > /var/lib/rancher/rke2/server/manifests/01-longhorn.yaml << EOM -apiVersion: helm.cattle.io/v1 -kind: HelmChart -metadata: - name: longhorn - namespace: kube-system -spec: - chart: longhorn - repo: https://charts.longhorn.io - version: 1.7.1 - targetNamespace: kube-system -EOM - -#metallb helm values: https://github.com/metallb/metallb/tree/main/charts/metallb -cat > /var/lib/rancher/rke2/server/manifests/02-metallb.yaml << EOM -apiVersion: helm.cattle.io/v1 -kind: HelmChart -metadata: - name: metallb - namespace: kube-system -spec: - chart: metallb - repo: https://metallb.github.io/metallb - version: 0.14.8 - targetNamespace: kube-system -EOM +envsubst < helmchart-template.yaml > /var/lib/rancher/rke2/server/manifests/00-helmcharts.yaml +# We install longhorn from a template to avoid install issues with the HelmController +# +LONGHORN_VERSION=1.8.1 +HELM_LATEST=$(curl -L --silent --show-error --fail "https://get.helm.sh/helm-latest-version" 2>&1 || true) +curl https://get.helm.sh/helm-$HELM_LATEST-linux-amd64.tar.gz --output helm.tar.gz +tar -xvf ./helm.tar.gz && rm -rf ./helm.tar.gz +chmod +x ./linux-amd64/helm +./linux-amd64/helm repo add longhorn https://charts.longhorn.io +./linux-amd64/helm repo update +./linux-amd64/helm template longhorn longhorn/longhorn --version $LONGHORN_VERSION --set defaultSettings.deletingConfirmationFlag=true --set longhornUI.replicas=0 --set namespaceOverride=kube-system --no-hooks > /var/lib/rancher/rke2/server/manifests/01-longhorn.yaml +rm -rf ./linux-amd64 info "Installing awscli" yum install -y unzip jq || apt-get -y install unzip jq curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" unzip awscliv2.zip sudo ./aws/install +curl -L https://github.com/mikefarah/yq/releases/download/v4.40.4/yq_linux_amd64 -o yq +chmod +x yq echo "Getting OIDC keypair" sudo mkdir /irsa @@ -84,19 +50,18 @@ aws secretsmanager get-secret-value --secret-id ${secret_prefix}-oidc-public-key chcon -t svirt_sandbox_file_t /irsa/* info "Setting up RKE2 config file" -curl -L https://github.com/mikefarah/yq/releases/download/v4.40.4/yq_linux_amd64 -o yq -chmod +x yq ./yq -i '.cloud-provider-name += "external"' /etc/rancher/rke2/config.yaml ./yq -i '.disable-cloud-controller += "true"' /etc/rancher/rke2/config.yaml ./yq -i '.kube-apiserver-arg += "service-account-key-file=/irsa/signer.key.pub"' /etc/rancher/rke2/config.yaml -./yq -i '.kube-apiserver-arg += "service-account-key-file=/irsa/signer.key.pub"' /etc/rancher/rke2/config.yaml ./yq -i '.kube-apiserver-arg += "service-account-signing-key-file=/irsa/signer.key"' /etc/rancher/rke2/config.yaml ./yq -i '.kube-apiserver-arg += "api-audiences=kubernetes.svc.default"' /etc/rancher/rke2/config.yaml ./yq -i '.kube-apiserver-arg += "service-account-issuer=https://${BUCKET_REGIONAL_DOMAIN_NAME}"' /etc/rancher/rke2/config.yaml ./yq -i '.kube-apiserver-arg += "audit-log-path=/var/log/kubernetes/audit/audit.log"' /etc/rancher/rke2/config.yaml +#Fix for metrics server scraping of kubernetes api server components +./yq -i '.kube-controller-manager-arg[2] = "bind-address=0.0.0.0"' /etc/rancher/rke2/config.yaml +./yq -i '.kube-scheduler-arg += "bind-address=0.0.0.0"' /etc/rancher/rke2/config.yaml +./yq -i '.etcd-arg += "listen-metrics-urls=http://0.0.0.0:2381"|.etcd-arg style="double"' /etc/rancher/rke2/config.yaml rm -rf ./yq - - } pre_userdata diff --git a/.github/workflows/test-aks.yaml b/.github/workflows/test-aks.yaml index 49eec97a0a..de813d5326 100644 --- a/.github/workflows/test-aks.yaml +++ b/.github/workflows/test-aks.yaml @@ -88,12 +88,19 @@ jobs: - name: Create IAC run: uds run -f tasks/iac.yaml apply-tofu --no-progress --set K8S_DISTRO=aks --set CLOUD=azure + - name: Configure Cluster DNS + run: uds run -f tasks/utils.yaml aks-coredns-setup --no-progress + - name: Deploy Core Bundle env: UDS_CONFIG: .github/bundles/aks/uds-config.yaml run: uds deploy .github/bundles/aks/uds-bundle-uds-core-aks-nightly-*.tar.zst --confirm timeout-minutes: 30 + - name: Test UDS Core + run: uds run -f tasks/test.yaml uds-core-non-k3d --set EXCLUDED_PACKAGES="metrics-server" + continue-on-error: true + - name: Debug Output if: ${{ always() }} uses: ./.github/actions/debug-output diff --git a/.github/workflows/test-eks.yaml b/.github/workflows/test-eks.yaml index 1ea46e8f54..a7d7977b3f 100644 --- a/.github/workflows/test-eks.yaml +++ b/.github/workflows/test-eks.yaml @@ -92,12 +92,19 @@ jobs: run: uds run -f tasks/iac.yaml create-iac --no-progress --set K8S_DISTRO=eks --set CLOUD=aws timeout-minutes: 20 + - name: Configure Cluster DNS + run: uds run -f tasks/utils.yaml eks-coredns-setup --no-progress + - name: Deploy Core Bundle env: UDS_CONFIG: .github/bundles/eks/uds-config.yaml run: uds deploy .github/bundles/eks/uds-bundle-uds-core-eks-nightly-*.tar.zst --confirm timeout-minutes: 30 + - name: Test UDS Core + run: uds run -f tasks/test.yaml uds-core-non-k3d --set EXCLUDED_PACKAGES="metrics-server" + continue-on-error: true + - name: Debug Output if: ${{ always() }} uses: ./.github/actions/debug-output diff --git a/.github/workflows/test-rke2.yaml b/.github/workflows/test-rke2.yaml index 140ee8ee6b..8bb0c9f234 100644 --- a/.github/workflows/test-rke2.yaml +++ b/.github/workflows/test-rke2.yaml @@ -101,6 +101,10 @@ jobs: run: uds deploy .github/bundles/rke2/uds-bundle-uds-core-rke2-nightly-*.tar.zst --confirm timeout-minutes: 30 + - name: Test UDS Core + run: uds run -f tasks/test.yaml uds-core-non-k3d + continue-on-error: true + - name: Debug Output if: ${{ always() }} uses: ./.github/actions/debug-output diff --git a/src/istio/tasks.yaml b/src/istio/tasks.yaml index ab7f2ec080..3344957c7e 100644 --- a/src/istio/tasks.yaml +++ b/src/istio/tasks.yaml @@ -6,7 +6,7 @@ tasks: inputs: validate_passthrough: description: Whether to validate the passthrough gateway - default: "true" + actions: - description: Validate the Istio Admin Gateway wait: diff --git a/src/test/tasks.yaml b/src/test/tasks.yaml index 49ff58e275..4b35fb5a6e 100644 --- a/src/test/tasks.yaml +++ b/src/test/tasks.yaml @@ -1,6 +1,9 @@ # Copyright 2024 Defense Unicorns # SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial +includes: + - utils: ../../tasks/utils.yaml + tasks: - name: validate actions: @@ -24,9 +27,14 @@ tasks: - name: create-deploy description: Test app used for UDS Core validation + inputs: + architecture: + description: "System architecture that the test-apps package should be built for." + default: ${UDS_ARCH} + actions: - description: Create zarf package for the test resources - cmd: "uds zarf package create src/test --confirm --no-progress --skip-sbom" + cmd: uds zarf package create src/test --confirm --no-progress --skip-sbom -a ${{ index .inputs "architecture" }} - description: Deploy the test resources cmd: "uds zarf package deploy build/zarf-package-uds-core-test-apps-*.zst --confirm --no-progress" @@ -120,9 +128,10 @@ tasks: - description: Verify the authservice tenant app is protected by checking redirect maxRetries: 3 + task: utils:tenant-gw-ip cmd: | set -e - SSO_REDIRECT=$(uds zarf tools kubectl run curl-test --image=cgr.dev/chainguard/curl:latest -q --restart=Never --rm -i -- -Ls -o /dev/null -w %{url_effective} "https://protected.uds.dev") + SSO_REDIRECT=$(uds zarf tools kubectl run curl-test --image=cgr.dev/chainguard/curl:latest -q --restart=Never --rm -i -- --resolve 'protected.uds.dev:$TENANT_GW_IP:443' -Ls -o /dev/null -w %{url_effective} "https://protected.uds.dev") case "${SSO_REDIRECT}" in "https://sso.uds.dev"*) diff --git a/src/velero/tasks.yaml b/src/velero/tasks.yaml index 1394e5f8b6..8861d2e0e3 100644 --- a/src/velero/tasks.yaml +++ b/src/velero/tasks.yaml @@ -47,12 +47,12 @@ tasks: - description: wait for the backup object wait: cluster: - kind: Backup + kind: backup.velero.io name: ${BACKUP_NAME} namespace: velero - description: check the status of the backup object cmd: |- - STATUS=$(uds zarf tools kubectl get backups -n velero ${BACKUP_NAME} -o jsonpath='{.status.phase}') + STATUS=$(uds zarf tools kubectl get backup.velero.io -n velero ${BACKUP_NAME} -o jsonpath='{.status.phase}') if [ ${STATUS} != "Completed" ]; then echo "Status is '$STATUS'... waiting to see if it changes" @@ -60,13 +60,13 @@ tasks: sleep 30 # check again... - STATUS=$(uds zarf tools kubectl get backups -n velero ${BACKUP_NAME} -o jsonpath='{.status.phase}') + STATUS=$(uds zarf tools kubectl get backup.velero.io -n velero ${BACKUP_NAME} -o jsonpath='{.status.phase}') if [ ${STATUS} != "Completed" ]; then echo "Status is $STATUS... something isn't right.." # get backup object - uds zarf tools kubectl get backups -n velero ${BACKUP_NAME} -o yaml - uds zarf tools kubectl get backups -A -o yaml + uds zarf tools kubectl get backup.velero.io -n velero ${BACKUP_NAME} -o yaml + uds zarf tools kubectl get backup.velero.io -A -o yaml echo "::endgroup::" # get backupstoragelocations diff --git a/tasks/iac.yaml b/tasks/iac.yaml index ac206cdbe7..d6ef7871ce 100644 --- a/tasks/iac.yaml +++ b/tasks/iac.yaml @@ -1,6 +1,8 @@ # Copyright 2024 Defense Unicorns # SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial +includes: + - util: ./utils.yaml variables: - name: CLUSTER_NAME @@ -109,27 +111,34 @@ tasks: actions: - task: rke2-nodes-ready - cmd: | - # wait for at least 3 nodes + # wait for nodes to be ready + echo "Waiting for all cluster nodes to be ready..."; while true; do - if [ $(uds zarf tools kubectl get nodes -o jsonpath='{range .items[*]}{.status.conditions[-1].type}={.status.conditions[-1].status}{"\n"}{end}' | egrep -i '^ready.*true' | wc -l) -lt 3 ]; then - echo "Waiting for at least 3 nodes to be ready..."; + if [ $(uds zarf tools kubectl get nodes | grep Ready | wc -l) -lt 4 ]; then sleep 5; else break fi done - # wait for cluster components + echo "Waiting for Helm Controller to install cluster components..." + while true; do + if [ $(uds zarf tools kubectl get po -n kube-system -l batch.kubernetes.io/controller-uid --no-headers | egrep -v Completed | wc -l) -gt 0 ]; then + sleep 5; + else + break + fi + done + echo "Waiting for cluster components to be ready..."; while true; do if [ $(uds zarf tools kubectl get po,job -A --no-headers=true | egrep -v 'Running|Complete' | wc -l) -gt 0 ]; then - echo "Waiting for cluster components to be ready..."; sleep 5; else - echo "Cluster is ready" + echo "Cluster is ready!" break fi done - uds zarf tools kubectl apply -f ./metallb.yaml + - task: util:rke2-allow-prom-kube-dns dir: .github/test-infra/aws/rke2/ maxTotalSeconds: 600 diff --git a/tasks/test.yaml b/tasks/test.yaml index 241ac8d8aa..a9e6f8f2ee 100644 --- a/tasks/test.yaml +++ b/tasks/test.yaml @@ -1,6 +1,11 @@ # Copyright 2024 Defense Unicorns # SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial +variables: + - name: EXCLUDED_PACKAGES + description: A comma separated string of packages to be excluded from validation and e2e tests. + default: "" + includes: - create: ./create.yaml - setup: ./setup.yaml @@ -53,11 +58,22 @@ tasks: - name: validate-packages description: "Validate all packages" + inputs: + # Added to support bypassing passthrough gateway validation on non-k3d distributions. + validate_passthrough: + description: Whether to validate the passthrough gateway. + default: "true" # loop through each src/* package and run the validate.yaml task actions: - cmd: | - for package in src/*; do - uds run -f ${package}/tasks.yaml validate --no-progress + for package in $(ls src); do + if [ ! $(echo ${EXCLUDED_PACKAGES} | grep ${package}) ]; then + if [ ${package} = "istio" ]; then + uds run -f src/${package}/tasks.yaml validate --no-progress --with validate_passthrough=${{ .inputs.validate_passthrough }} + else + uds run -f src/${package}/tasks.yaml validate --no-progress + fi + fi done set +e @@ -73,6 +89,11 @@ tasks: - name: e2e-tests description: "E2E Test all packages" + inputs: + architecture: + description: "System architecture that the test-apps package should be built for." + required: true + default: ${UDS_ARCH} # Run each e2e test type from the e2e folder actions: - description: "Setup the Keycloak admin user if needed" @@ -82,6 +103,8 @@ tasks: cmd: uds run -f tasks/test.yaml common-setup:create-doug-user --set KEYCLOAK_GROUP="/UDS Core/Admin" # Adds the test doug user - description: "Create and Deploy Test App Package" task: test-resources:create-deploy + with: + architecture: ${{ .inputs.architecture }} - description: "Run Playwright E2E tests for all packages" dir: test/playwright cmd: | @@ -114,6 +137,17 @@ tasks: - task: uds-core - task: e2e-tests + - name: uds-core-non-k3d + description: "Validate and Test UDS Core deployment on a non K3d Cluster" + actions: + - task: util:setup-hosts + - task: validate-packages + with: + validate_passthrough: "false" + - task: e2e-tests + with: + architecture: "amd64" + - name: uds-core-ha description: "Build and test UDS Core" actions: diff --git a/tasks/utils.yaml b/tasks/utils.yaml index 8ca6899d16..e1e4f1c78e 100644 --- a/tasks/utils.yaml +++ b/tasks/utils.yaml @@ -64,3 +64,121 @@ tasks: --from-literal=password=${PASSWORD} \ -n keycloak fi + - name: aks-coredns-setup + actions: + - description: Setup Custom ConfigMap for Core DNS + cmd: | + uds zarf tools kubectl apply -f - </dev/null) + if [ -z $IP_ADDR ]; then + HOSTNAME=$(uds zarf tools kubectl get service -n istio-admin-gateway admin-ingressgateway -o=jsonpath='{.status.loadBalancer.ingress[0].hostname}' 2>/dev/null) + IP_ADDR=$(dig +short $HOSTNAME) + fi; echo $IP_ADDR + mute: true + setVariables: + - name: ADMIN_GW_IP + - name: tenant-gw-ip + actions: + - description: Fetch Tenant Gateway IP Address + cmd: | + IP_ADDR=$(uds zarf tools kubectl get service -n istio-tenant-gateway tenant-ingressgateway -o=jsonpath='{.status.loadBalancer.ingress[0].ip}' 2>/dev/null) + if [ -z $IP_ADDR ]; then + HOSTNAME=$(uds zarf tools kubectl get service -n istio-tenant-gateway tenant-ingressgateway -o=jsonpath='{.status.loadBalancer.ingress[0].hostname}' 2>/dev/null) + IP_ADDR=$(dig +short $HOSTNAME) + fi; echo $IP_ADDR + mute: true + setVariables: + - name: TENANT_GW_IP + - name: setup-hosts + actions: + - task: admin-gw-ip + - task: tenant-gw-ip + - description: Adds Cluster LoadBalancer IP Addresses to match appropriate hosts names in /etc/hosts + mute: true + cmd: | + echo "$ADMIN_GW_IP keycloak.admin.uds.dev neuvector.admin.uds.dev grafana.admin.uds.dev demo.admin.uds.dev\n$TENANT_GW_IP sso.uds.dev demo-8080.uds.dev demo-8081.uds.dev protected.uds.dev" | sudo tee --append /etc/hosts diff --git a/test/jest/network.spec.ts b/test/jest/network.spec.ts index 93a78bcfc2..56c1ac8e86 100644 --- a/test/jest/network.spec.ts +++ b/test/jest/network.spec.ts @@ -23,6 +23,8 @@ function getCurlCommand(serviceName: string, namespaceName: string, port = 8080) return [ "curl", "-s", + "-m", + "3", "-o", "/dev/null", "-w", @@ -106,6 +108,23 @@ async function execInPod( }); } +// Check for HTTP error codes in test responses +// Used when checking if network calls were denied +// HTTP response status code reference: https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Status +// Expects curlOutput.stdout to only contain a string indicating the HTTP response code +function isResponseError(curlOutput: { stdout: string, stderr: string }) { + if (!curlOutput.stderr) { + const httpResponseCode = Number(curlOutput.stdout) ?? 0 + if (httpResponseCode < 100 || httpResponseCode > 399) { + return true + } else { + return false + } + } else { + return true + } +} + let curlPodName1 = ""; let testAdminApp = ""; let curlPodName6 = ""; @@ -134,6 +153,8 @@ describe("Network Policy Validation", () => { const GOOGLE_CURL = [ "curl", "-s", + "-m", + "3", "-o", "/dev/null", "-w", @@ -143,12 +164,13 @@ describe("Network Policy Validation", () => { test.concurrent("Denied Requests by Default and Incorrect Ports and Labels", async () => { // Default Deny when no Ingress or Egress defined or Exposed Endpoints + // The HTTP response code could either be 000 or 503, depending on the K8s distro const denied_external_response = await execInPod("curl-ns-deny-all-1", curlPodName1, "curl-pkg-deny-all-1", CURL_GATEWAY); - expect(denied_external_response.stdout).toBe("000"); + expect(isResponseError(denied_external_response)).toBe(true); // Default deny when no Ingress or Egress for internal curl command const denied_internal_response = await execInPod("curl-ns-deny-all-1", curlPodName1, "curl-pkg-deny-all-1", INTERNAL_CURL_COMMAND_1); - expect(denied_internal_response.stdout).toBe("503"); + expect(isResponseError(denied_internal_response)).toBe(true); // Default Deny for Google Curl when no Egress defined const denied_google_response = await execInPod("curl-ns-deny-all-1", curlPodName1, "curl-pkg-deny-all-1", GOOGLE_CURL); @@ -157,7 +179,7 @@ describe("Network Policy Validation", () => { // Default Deny for Blocked Port const blocked_port_curl = getCurlCommand("curl-pkg-deny-all-2", "curl-ns-deny-all-2", 9999); const denied_port_response = await execInPod("curl-ns-deny-all-1", curlPodName1, "curl-pkg-deny-all-1", blocked_port_curl); - expect(denied_port_response.stdout).toBe("503"); + expect(isResponseError(denied_port_response)).toBe(true); }); test.concurrent("Basic Wide Open Ingress and Wide Open Egress", async () => { @@ -168,6 +190,8 @@ describe("Network Policy Validation", () => { const CURL_INTERNAL_8081 = [ "curl", "-s", + "-m", + "3", "-o", "/dev/null", "-w", @@ -177,12 +201,12 @@ describe("Network Policy Validation", () => { // Deny request when port is not allowed on ingress const denied_incorrect_port_response = await execInPod("test-admin-app", testAdminApp, "curl", CURL_INTERNAL_8081); - expect(denied_incorrect_port_response.stdout).toBe("503"); + expect(isResponseError(denied_incorrect_port_response)).toBe(true); // Default Deny for undefined Ingress port const blocked_port_curl = getCurlCommand("curl-pkg-allow-all", "curl-ns-allow-all", 9999); const denied_port_response = await execInPod("test-admin-app", testAdminApp, "curl", blocked_port_curl); - expect(denied_port_response.stdout).toBe("503"); + expect(isResponseError(denied_port_response)).toBe(true); // Wide open Egress means successful google curl const successful_google_response = await execInPod("test-admin-app", testAdminApp, "curl", GOOGLE_CURL); @@ -233,7 +257,7 @@ describe("Network Policy Validation", () => { // Default Deny for Blocked Port const blocked_port_curl = getCurlCommand("curl-pkg-remote-ns-ingress", "curl-ns-remote-ns-2", 9999); const denied_port_response = await execInPod("curl-ns-remote-ns-1", curlPodName6, "curl-pkg-remote-ns-egress", blocked_port_curl); - expect(denied_port_response.stdout).toBe("503"); + expect(isResponseError(denied_port_response)).toBe(true); }); test.concurrent("Kube API Restrictions", async () => { @@ -254,7 +278,7 @@ describe("Network Policy Validation", () => { // Default Deny for Blocked Port const blocked_port_curl = getCurlCommand("curl-pkg-deny-all-2", "curl-ns-deny-all-2", 9999); const denied_port_response = await execInPod("curl-ns-kube-api", curlPodName8, "curl-pkg-kube-api", blocked_port_curl); - expect(denied_port_response.stdout).toBe("503"); + expect(isResponseError(denied_port_response)).toBe(true); }); test.concurrent("RemoteCidr Restrictions", async () => {