Release/v1.4 Remove AKS-Engine

.pipelines/cni/cilium/cilium-cni-load-test.yaml

@@ -16,7 +16,7 @@ stages:
               inlineScript: |
                 set -ex
                 make -C ./hack/swift azcfg AZCLI=az REGION=$(LOCATION)
-                make -C ./hack/swift overlay-byocni-up AZCLI=az REGION=$(LOCATION) SUB=$(SUBSCRIPTION_ID) CLUSTER=${RESOURCE_GROUP} NODE_COUNT=10 VM_SIZE=Standard_DS4_v2
+                make -C ./hack/swift overlay-no-kube-proxy-up AZCLI=az REGION=$(LOCATION) SUB=$(SUBSCRIPTION_ID) CLUSTER=${RESOURCE_GROUP}-$(make revision) NODE_COUNT=10 VM_SIZE=Standard_DS4_v2
             name: "CreateAksCluster"
             displayName: "Create AKS Cluster"
   - stage: install_cilium
@@ -35,7 +35,7 @@ stages:
               inlineScript: |
                 set -ex
                 az extension add --name aks-preview
-                make -C ./hack/swift set-kubeconf AZCLI=az CLUSTER=${RESOURCE_GROUP}
+                make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(make revision)
                 ls -lah
                 pwd
                 kubectl cluster-info
@@ -48,6 +48,9 @@ stages:
                 echo "install Cilium onto Overlay Cluster"
                 kubectl apply -f test/integration/manifests/cilium/cilium-agent
                 kubectl apply -f test/integration/manifests/cilium/cilium-operator
+                # Passes Cilium image to daemonset and deployment
+                envsubst '${CILIUM_VERSION_TAG},${CILIUM_IMAGE_REGISTRY}' < test/integration/manifests/cilium/daemonset.yaml | kubectl apply -f -
+                envsubst '${CILIUM_VERSION_TAG},${CILIUM_IMAGE_REGISTRY}' < test/integration/manifests/cilium/deployment.yaml | kubectl apply -f -
                 kubectl get po -owide -A
                 echo "deploy ip-masq-agent for overlay"
                 kubectl create -f test/integration/manifests/ip-masq-agent/ip-masq-agent.yaml --validate=false
@@ -62,20 +65,11 @@ stages:
     jobs:
       - job: deploy_pods
         steps:
-          - task: AzureCLI@1
-            displayName: "Pod Deployment"
-            inputs:
-              azureSubscription: $(TEST_SUB_SERVICE_CONNECTION)
-              scriptLocation: "inlineScript"
-              scriptType: "bash"
-              addSpnToEnvironment: true
-              inlineScript: |
-                set -ex
-                az extension add --name aks-preview
-                make -C ./hack/swift set-kubeconf AZCLI=az CLUSTER=${RESOURCE_GROUP}
-                kubectl apply -f hack/manifests/pod.yaml
-                kubectl apply -f hack/manifests/hostprocess.yaml
-                bash hack/scripts/scale_deployment.sh
+          - template: ../load-test-templates/pod-deployment-template.yaml
+            parameters:
+              clusterName: ${{ parameters.clusterName }}
+              scaleup: 2400
+              os: linux
   - stage: validate_state
     dependsOn: pod_deployment
     displayName: "Validate State"
@@ -89,9 +83,9 @@ stages:
               scriptType: "bash"
               addSpnToEnvironment: true
               inlineScript: |
-                make -C ./hack/swift set-kubeconf AZCLI=az CLUSTER=${RESOURCE_GROUP}
+                make -C ./hack/swift set-kubeconf AZCLI=az CLUSTER=${RESOURCE_GROUP}-$(make revision)
                 kubectl get pods -A
-                bash hack/scripts/validate_state.sh
+                make test-validate-state
             name: "ValidateState"
             displayName: "Validate State"
             retryCountOnTaskFailure: 3
@@ -109,18 +103,22 @@ stages:
               addSpnToEnvironment: true
               inlineScript: |
                 echo "Scale up the pods and immediated restart the nodes"
-                make -C ./hack/swift set-kubeconf AZCLI=az CLUSTER=${RESOURCE_GROUP}
+                make -C ./hack/swift set-kubeconf AZCLI=az CLUSTER=${RESOURCE_GROUP}-$(make revision)
                 make -C ./hack/swift azcfg AZCLI=az REGION=$(LOCATION)
+                cd test/integration/load
                 echo "Scaling the pods down to 100 per node"
-                bash ./hack/scripts/scale_deployment.sh -n 0 -u 1000 -s
+                go test -count 1 -timeout 30m -tags load -run ^TestScaleDeployment$ -tags=load -scaleup=1000 -skip-wait=true
+                cd ../../../
                 echo "Restarting the nodes"
                 vmss_name=$(az vmss list -g MC_${RESOURCE_GROUP}_${RESOURCE_GROUP}_$(LOCATION) --query "[].name" -o tsv)
-                make -C ./hack/swift restart-vmss AZCLI=az CLUSTER=${RESOURCE_GROUP} REGION=$(LOCATION) VMSS_NAME=$vmss_name
-                bash ./hack/scripts/scale_deployment.sh -n 0 -u 1000 -c
+                make -C ./hack/swift restart-vmss AZCLI=az CLUSTER=${RESOURCE_GROUP}-$(make revision) REGION=$(LOCATION) VMSS_NAME=$vmss_name
+                cd test/integration/load
+                go test -count 1 -timeout 30m -tags load -run ^TestScaleDeployment$ -tags=load -replicas=1000
             name: "RestartNodes"
             displayName: "Restart Nodes"
           - script: |
-              bash hack/scripts/validate_state.sh
+              export RESTART_CASE=true
+              make test-validate-state
             name: "ValidateState"
             displayName: "Validate State"
             retryCountOnTaskFailure: 3
@@ -148,11 +146,11 @@ stages:
               addSpnToEnvironment: true
               inlineScript: |
                 set -ex
-                make -C ./hack/swift set-kubeconf AZCLI=az CLUSTER=${RESOURCE_GROUP}
+                make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(make revision)
             name: "GetCluster"
             displayName: "Get AKS Cluster"
           - script: |
-              k delete deployment container -n default
+              kubectl delete ns load-test
               cilium connectivity test
             retryCountOnTaskFailure: 6
             name: "CiliumConnectivityTests"
@@ -175,9 +173,9 @@ stages:
                 if [ "$(DELETE_RESOURCES)" ]
                 then
                   echo "Deleting Cluster and resource group"
-                  make -C ./hack/swift set-kubeconf AZCLI=az CLUSTER=${RESOURCE_GROUP}
-                  make -C ./hack/swift azcfg AZCLI=az REGION=$(LOCATION)
-                  make -C ./hack/swift down AZCLI=az REGION=$(LOCATION) SUB=$(SUBSCRIPTION_ID) CLUSTER=${RESOURCE_GROUP}
+                  make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(make revision)
+                  make -C ./hack/aks azcfg AZCLI=az REGION=$(LOCATION)
+                  make -C ./hack/aks down AZCLI=az REGION=$(LOCATION) SUB=$(SUBSCRIPTION_ID) CLUSTER=${{ parameters.clusterName }}-$(make revision)
                   echo "Cluster and resources down"
                 else
                   echo "Deletion of resources is False"

.pipelines/cni/cilium/nightly-release-test.yml

@@ -0,0 +1,54 @@
+pr: none
+trigger: none
+
+stages:
+  - stage: init
+    displayName: "Build and Push Cilium Image"
+    jobs:
+      - job: build_and_push_cilium_image
+        steps:
+          - bash: |
+              set -ex
+              cd .pipelines/
+              git clone https://github.com/cilium/cilium.git
+              cd cilium
+              make docker-cilium-image
+              make docker-operator-generic-image
+            name: BuildCiliumImage
+            displayName: "Build Cilium Image"
+          - task: Docker@2
+            displayName: Login
+            inputs:
+              containerRegistry: $(CONTAINER_REGISTRY)
+              command: "login"
+          - task: AzureCLI@1
+            inputs:
+              azureSubscription: $(TEST_SUB_SERVICE_CONNECTION)
+              scriptLocation: "inlineScript"
+              inlineScript: |
+                set -ex
+                docker tag quay.io/cilium/cilium:$(DOCKER_IMAGE_TAG) $(ACR).azurecr.io/cilium/cilium:$(DOCKER_IMAGE_TAG)
+                docker tag quay.io/cilium/operator-generic:$(DOCKER_IMAGE_TAG) $(ACR).azurecr.io/cilium/operator-generic:$(DOCKER_IMAGE_TAG)
+                docker push $(ACR).azurecr.io/cilium/cilium:$(DOCKER_IMAGE_TAG)
+                docker push $(ACR).azurecr.io/cilium/operator-generic:$(DOCKER_IMAGE_TAG)
+            name: "PushCiliumImage"
+            displayName: "Push Cilium Image"
+          - task: Docker@2
+            displayName: Logout
+            inputs:
+              containerRegistry: $(CONTAINER_REGISTRY)
+              command: "logout"
+  - stage: cilium_nightly
+    displayName: E2E - Cilium Nightly
+    variables:
+      GOPATH: "$(Agent.TempDirectory)/go" # Go workspace path
+      GOBIN: "$(GOPATH)/bin" # Go binaries path
+      modulePath: "$(GOPATH)/src/github.com/Azure/azure-container-networking"
+    jobs:
+      - job: cilium_nightly
+        steps:
+        - template: ../../singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml
+          parameters:
+            name: "cilium_nightly"
+            testDropgz: ""
+            clusterName: "ciliumnightly"

.pipelines/cni/k8s-e2e/k8s-e2e-job-template.yaml

@@ -0,0 +1,111 @@
+parameters:
+  clusterName: ""
+  os: ""
+  dependsOn: ""
+  sub: ""
+
+
+jobs:
+  - job: cni_k8se2e
+    displayName: "CNI k8s E2E"
+    dependsOn: ${{ parameters.dependsOn }}
+    pool:
+      name: $(BUILD_POOL_NAME_DEFAULT)
+    steps:
+      - task: AzureCLI@1
+        inputs:
+          azureSubscription: ${{ parameters.sub }}
+          scriptLocation: "inlineScript"
+          scriptType: "bash"
+          addSpnToEnvironment: true
+          inlineScript: |
+            set -e
+            make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(make revision)
+
+            # sig-release provides test suite tarball(s) per k8s release. Just need to provide k8s version "v1.xx.xx"
+            # pulling k8s version from AKS.
+            eval k8sVersion="v"$( az aks show -g ${{ parameters.clusterName }}-$(make revision) -n ${{ parameters.clusterName }}-$(make revision) --query "currentKubernetesVersion")
+            curl -L https://dl.k8s.io/$k8sVersion/kubernetes-test-linux-amd64.tar.gz -o ./kubernetes-test-linux-amd64.tar.gz
+
+            # https://github.com/kubernetes/sig-release/blob/master/release-engineering/artifacts.md#content-of-kubernetes-test-system-archtargz-on-example-of-kubernetes-test-linux-amd64targz-directories-removed-from-list
+            # explictly unzip and strip directories from ginkgo and e2e.test
+            tar -xvzf kubernetes-test-linux-amd64.tar.gz --strip-components=3 kubernetes/test/bin/ginkgo kubernetes/test/bin/e2e.test
+
+        displayName: "Setup Environment"
+      - ${{ if eq(parameters.datapath, true) }}:
+        - template: ../k8s-e2e/k8s-e2e-step-template.yaml
+          parameters:
+            testName: Datapath
+            name: datapath
+            clusterName: ${{ parameters.clusterName }}
+            ginkgoFocus: '(.*).Networking.should|(.*).Networking.Granular|(.*)kubernetes.api'
+            ginkgoSkip: 'SCTP|Disruptive|Slow|hostNetwork|kube-proxy|IPv6'
+            os: ${{ parameters.os }}
+            processes: 8
+            attempts: 10
+      - ${{ if eq(parameters.dns, true) }}:
+        - template: ../k8s-e2e/k8s-e2e-step-template.yaml
+          parameters:
+            testName: DNS
+            name: dns
+            clusterName: ${{ parameters.clusterName }}
+            ginkgoFocus: '\[sig-network\].DNS.should'
+            ginkgoSkip: 'resolv|256 search'
+            os: ${{ parameters.os }}
+            processes: 8
+            attempts: 3
+      - ${{ if eq(parameters.portforward, true) }}:
+        - template: ../k8s-e2e/k8s-e2e-step-template.yaml
+          parameters:
+            testName: Kubectl Portforward
+            name: portforward
+            clusterName: ${{ parameters.clusterName }}
+            ginkgoFocus: '\[sig-cli\].Kubectl.Port'
+            ginkgoSkip: ''
+            os: ${{ parameters.os }}
+            processes: 8
+            attempts: 3
+      - ${{ if eq(parameters.loadBalancer, true) }}:
+        - template: ../k8s-e2e/k8s-e2e-step-template.yaml
+          parameters:
+            testName: Load Balancers
+            name: load
+            clusterName: ${{ parameters.clusterName }}
+            ginkgoFocus: '\[sig-network\].LoadBalancers'
+            ginkgoSkip: 'ESIPP|Serial'
+            os: ${{ parameters.os }}
+            processes: 8
+            attempts: 3
+      - ${{ if eq(parameters.service, true) }}:
+        - template: ../k8s-e2e/k8s-e2e-step-template.yaml
+          parameters:
+            testName: Service Conformance
+            name: service
+            clusterName: ${{ parameters.clusterName }}
+            ginkgoFocus: 'Services.*\[Conformance\].*'
+            ginkgoSkip: ''
+            os: ${{ parameters.os }}
+            processes: 8
+            attempts: 3
+      - ${{ if eq(parameters.hostport, true) }}:
+        - template: ../k8s-e2e/k8s-e2e-step-template.yaml
+          parameters:
+            testName: Host Port
+            name: hostport
+            clusterName: ${{ parameters.clusterName }}
+            ginkgoFocus: '\[sig-network\](.*)HostPort|\[sig-scheduling\](.*)hostPort'
+            ginkgoSkip: 'SCTP|exists conflict' # Skip slow 5 minute test
+            os: ${{ parameters.os }}
+            processes: 1 # Has a short serial test
+            attempts: 3
+      - ${{ if and(eq(parameters.hybridWin, true), eq(parameters.os, 'windows')) }}:
+        - template: ../k8s-e2e/k8s-e2e-step-template.yaml
+          parameters:
+            testName: Hybrid Network
+            name: hybrid
+            clusterName: ${{ parameters.clusterName }}
+            ginkgoFocus: '\[sig-windows\].Hybrid'
+            ginkgoSkip: ''
+            os: ${{ parameters.os }}
+            processes: 8
+            attempts: 3

.pipelines/cni/k8s-e2e/k8s-e2e-step-template.yaml

@@ -0,0 +1,53 @@
+parameters:
+  testName: ""
+  name: ""
+  clusterName: ""
+  ginkgoFocus: ""
+  ginkgoSkip: ""
+  os: ""
+  processes: "" # Number of parallel processes
+  attempts: ""
+
+
+steps:
+  - script: |
+      set -ex
+
+      # ginkgoSkip cant handle only |LinuxOnly. Need to have check
+      if ${{ lower(and(ge(length(parameters.ginkgoSkip), 1), eq(parameters.os, 'windows'))) }}
+      then
+        SKIP="|LinuxOnly"
+      elif ${{ lower(eq(parameters.os, 'windows')) }}
+      then
+        SKIP="LinuxOnly"
+      fi
+
+      # Taint Linux nodes so that windows tests do not run on them
+      if ${{ lower(eq(parameters.os, 'windows')) }}
+      then
+        kubectl taint nodes -l kubernetes.azure.com/mode=system node-role.kubernetes.io/control-plane:NoSchedule
+      fi
+
+      # Depreciating flags. Change once k8s minimum version supported is > 1.24
+      # nodes -> procs
+      # flakeAttempts -> flake-attempts
+      # dryRun -> dry-run
+
+      ./ginkgo --nodes=${{ parameters.processes }} \
+      ./e2e.test -- \
+      --num-nodes=2 \
+      --provider=skeleton \
+      --ginkgo.focus='${{ parameters.ginkgoFocus }}' \
+      --ginkgo.skip="${{ parameters.ginkgoSkip }}$SKIP" \
+      --ginkgo.flakeAttempts=${{ parameters.attempts }} \
+      --ginkgo.v \
+      --node-os-distro=${{ parameters.os }} \
+      --kubeconfig=$HOME/.kube/config
+
+      # Untaint Linux nodes once testing is complete
+      if ${{ lower(eq(parameters.os, 'windows')) }}
+      then
+        kubectl taint nodes -l kubernetes.azure.com/mode=system node-role.kubernetes.io/control-plane:NoSchedule-
+      fi
+    name: ${{ parameters.name }}
+    displayName: k8s E2E - ${{ parameters.testName }}

.pipelines/cni/load-test-templates/create-cluster-template.yaml

@@ -0,0 +1,20 @@
+parameters:
+  clusterType: ""
+  clusterName: ""
+  nodeCount: ""
+  vmSize: ""
+  windowsVMSize: ""
+
+steps:
+  - task: AzureCLI@1
+    inputs:
+      azureSubscription: $(TEST_SUB_SERVICE_CONNECTION)
+      scriptLocation: "inlineScript"
+      scriptType: "bash"
+      addSpnToEnvironment: true
+      inlineScript: |
+        set -ex
+        make -C ./hack/aks azcfg AZCLI=az REGION=$(LOCATION)
+        make -C ./hack/aks ${{ parameters.clusterType }} AZCLI=az REGION=$(LOCATION) SUB=$(SUBSCRIPTION_ID) CLUSTER=${{ parameters.clusterName }}-$(make revision) NODE_COUNT=${{ parameters.nodeCount }} VM_SIZE=${{ parameters.vmSize }} WINDOWS_VM_SKU=${{ parameters.windowsVMSize }} WINDOWS_USERNAME=${WINDOWS_USERNAME} WINDOWS_PASSWORD=${WINDOWS_PASSWORD}
+    name: "CreateAksCluster"
+    displayName: "Create AKS Cluster"

.pipelines/cni/load-test-templates/pod-deployment-template.yaml

@@ -0,0 +1,20 @@
+parameters:
+  clusterName: ""
+  scaleup: 1000
+  os: ""
+  iterations: 4
+
+steps:
+  - task: AzureCLI@1
+    displayName: "Pod Deployment"
+    inputs:
+      azureSubscription: $(TEST_SUB_SERVICE_CONNECTION)
+      scriptLocation: "inlineScript"
+      scriptType: "bash"
+      addSpnToEnvironment: true
+      inlineScript: |
+        set -ex
+        az extension add --name aks-preview
+        make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(make revision)
+        cd test/integration/load
+        go test -timeout 30m -tags load -run ^TestLoad$ -tags=load -iterations=${{ parameters.iterations }} -scaleup=${{ parameters.scaleup }} -os=${{ parameters.os }}