Swiftv2 e2e tests

.pipelines/multitenancy/swiftv2-e2e-job-template.yaml

@@ -0,0 +1,84 @@
+parameters:
+  name: ""
+  displayName: ""
+  clusterType: ""
+  clusterName: ""
+  vmSize: ""
+  k8sVersion: ""
+  dependsOn: ""
+  nodePoolName: ""
+
+stages:
+  - stage: ${{ parameters.clusterName }}
+    displayName: Create Cluster - ${{ parameters.displayName }}
+    dependsOn:
+      - ${{ parameters.dependsOn }}
+      - setup
+    pool:
+      name: $(BUILD_POOL_NAME_DEFAULT)
+    variables:
+      commitID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.commitID'] ]
+    jobs:
+      - template: ../templates/create-cluster-swiftv2.yaml
+        parameters:
+          name: ${{ parameters.name }}
+          displayName: ${{ parameters.displayName }}
+          clusterType: ${{ parameters.clusterType }}
+          clusterName: ${{ parameters.clusterName }}-$(commitID)
+          vmSize: ${{ parameters.vmSize }}
+          k8sVersion: ${{ parameters.k8sVersion }}
+          dependsOn: ${{ parameters.dependsOn }}
+          region: $(REGION_SWIFTV2_CLUSTER_TEST) # Swiftv2 has a specific region requirements
+
+  - stage: ${{ parameters.name }}
+    # condition: and( succeeded(), not(eq(dependencies.dualstackoverlaye2e.result,'SucceededWithIssues')) ) # Cant use parameters in dependencies
+    displayName: E2E - ${{ parameters.displayName }}
+    dependsOn:
+    - setup
+    - publish
+    - ${{ parameters.clusterName }}
+    variables:
+      GOPATH: "$(Agent.TempDirectory)/go" # Go workspace path
+      GOBIN: "$(GOPATH)/bin" # Go binaries path
+      modulePath: "$(GOPATH)/src/github.com/Azure/azure-container-networking"
+      commitID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.commitID'] ]
+    pool:
+      name: $(BUILD_POOL_NAME_DEFAULT)
+    jobs:
+      - job: ${{ parameters.name }}
+        displayName: Swiftv2 Multitenancy E2E Test Suite - (${{ parameters.name }})
+        pool:
+          name: $(BUILD_POOL_NAME_DEFAULT)
+          demands:
+          - agent.os -equals Linux
+          - Role -equals $(CUSTOM_E2E_ROLE)
+        steps:
+          - template: swiftv2-e2e-step-template.yaml
+            parameters:
+              name: ${{ parameters.name }}
+              clusterName: ${{ parameters.clusterName }}-$(commitID)
+              os: linux
+
+      - template: ../cni/k8s-e2e/k8s-e2e-job-template.yaml
+        parameters:
+          sub: $(ACN_TEST_SERVICE_CONNECTION)
+          clusterName: ${{ parameters.clusterName }}-$(commitID)
+          os: ${{ parameters.os }}
+          dependsOn: ${{ parameters.name }}
+          datapath: true
+          dns: true
+          portforward: true
+          hostport: true
+          service: true
+
+      - job: failedE2ELogs
+        displayName: "Failure Logs"
+        dependsOn:
+          - ${{ parameters.name }}
+        condition: failed()
+        steps:
+          - template: ../templates/log-template.yaml
+            parameters:
+              clusterName: ${{ parameters.clusterName }}-$(commitID)
+              os: linux
+              cni: cniv2

.pipelines/multitenancy/swiftv2-e2e-step-template.yaml

@@ -0,0 +1,97 @@
+parameters:
+  name: ""
+  clusterName: ""
+  cni: cniv2
+  os: ""
+
+steps:
+  - bash: |
+      go version
+      go env
+      mkdir -p '$(GOBIN)'
+      mkdir -p '$(GOPATH)/pkg'
+      mkdir -p '$(modulePath)'
+      echo '##vso[task.prependpath]$(GOBIN)'
+      echo '##vso[task.prependpath]$(GOROOT)/bin'
+    name: "GoEnv"
+    displayName: "Set up the Go environment"
+
+  - task: KubectlInstaller@0
+    inputs:
+      kubectlVersion: latest
+
+  - task: AzureCLI@1
+    inputs:
+      azureSubscription: $(ACN_TEST_SERVICE_CONNECTION)
+      scriptLocation: "inlineScript"
+      scriptType: "bash"
+      addSpnToEnvironment: true
+      inlineScript: |
+        set -e
+        make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}
+        ls -lah
+        pwd
+        kubectl cluster-info
+        kubectl get po -owide -A
+        echo "Apply the pod network yaml to start the delegation"
+        less test/integration/manifests/swiftv2/podnetwork.yaml
+        envsubst '${SUBNET_TOKEN},${SUBNET_GUID},${SUBNET_RESOURCE_ID},${VNET_GUID}' < test/integration/manifests/swiftv2/podnetwork.yaml | kubectl apply -f -
+        echo "Check the podnetwork yaml file"
+        less test/integration/manifests/swiftv2/podnetwork.yaml
+        kubectl get pn
+        kubectl describe pn
+        echo "Apply the pod network instance yaml to reserve IP"
+        kubectl apply -f test/integration/manifests/swiftv2/pni.yaml
+        kubectl get pni
+        kubectl describe pni
+        export NODE_NAME_0="$(kubectl get nodes -o json | jq -r .items[0].metadata.name)"
+        echo $NODE_NAME_0
+        echo "Start the first pod using the reserved IP"
+        envsubst '$NODE_NAME_0' < test/integration/manifests/swiftv2/mtpod0.yaml | kubectl apply -f -
+        export NODE_NAME_1="$(kubectl get nodes -o json | jq -r .items[1].metadata.name)"
+        echo $NODE_NAME_1
+        echo "Start another pod using the reserved IP"
+        envsubst '$NODE_NAME_1' < test/integration/manifests/swiftv2/mtpod1.yaml | kubectl apply -f -
+        sleep 2m
+        kubectl get pod -o wide -A
+        sleep 2m
+        echo "Check pods after 4 minutes"
+        kubectl get po -owide -A
+        kubectl describe pni
+    name: "start_swiftv2_pods"
+    displayName: "Start Swiftv2 Pods"
+    env:
+      SUBNET_TOKEN: $(SUBNET_TOKEN)
+
+  - script: |
+      set -e
+      kubectl get po -owide -A
+      echo "Start the connection test"
+      kubectl exec mtpod0 -it -- ip a
+      echo "Check mtpnc"
+      kubectl get mtpnc
+      kubectl describe mtpnc mtpod0
+      kubectl describe mtpnc mtpod1
+      echo "Test the connection to a mtpod in the same delegated network and in another node"
+      export IP0=`kubectl describe mtpnc mtpod0 | grep "Primary IP" | cut -c 17- | head -c -4`
+      echo $IP0
+      kubectl exec mtpod0 -it -- ping -c 3 $IP0
+      export IP1=`kubectl describe mtpnc mtpod1 | grep "Primary IP" | cut -c 17- | head -c -4`
+      echo $IP1
+      kubectl exec mtpod0 -it -- ping -c 3 $IP1
+      echo "Test the connection to a non-mt pod (172.25.0.7) in the same delegated network and in another cluster"
+      kubectl exec mtpod0 -it -- ping -c 3 -W 1 172.25.0.7
+    retryCountOnTaskFailure: 3
+    name: "Swiftv2_Tests"
+    displayName: "Swiftv2 Tests through script"
+
+  - script: |
+      set -e
+      kubectl get po -owide -A
+      cd test/integration/swiftv2
+      echo "TestSwiftv2PodToPod and will run it after migration from scripts."
+      go test -count=1 swiftv2_test.go -timeout 3m -tags swiftv2 -run ^TestSwiftv2PodToPod$ -tags=swiftv2,integration -v
+    retryCountOnTaskFailure: 3
+    name: "Swiftv2_Tests_future_version"
+    displayName: "Swiftv2 Tests through code"
+

.pipelines/pipeline.yaml

@@ -504,6 +504,21 @@ stages:
         vmSize: Standard_B2ms
         dependsOn: "test"
 
+    # Swiftv2 E2E tests with multitenancy cluster start up
+    - template: multitenancy/swiftv2-e2e-job-template.yaml
+      parameters:
+        name: "swiftv2_e2e"
+        displayName: Swiftv2 Multitenancy
+        os: linux
+        clusterType: swiftv2-multitenancy-cluster-up
+        clusterName: "mtacluster"
+        nodePoolName: "mtapool"
+        vmSize: Standard_D4_v2
+        dependsOn: "test"
+        dummyClusterName: "swiftv2dummy"
+        dummyClusterType: "swiftv2-dummy-cluster-up"
+        dummyClusterDisplayName: Swiftv2 Multitenancy Dummy Cluster
+
     - stage: delete
       displayName: Delete Clusters
       condition: always()
@@ -518,6 +533,7 @@ stages:
         - aks_windows_22_e2e
         - dualstackoverlay_e2e
         - cilium_dualstackoverlay_e2e
+        - swiftv2_e2e
       variables:
         commitID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.commitID'] ]
       jobs:
@@ -566,6 +582,12 @@ stages:
               cilium_dualstackoverlay_e2e:
                 name: cilium_dualstackoverlay_e2e
                 clusterName: "cildsovere2e"
+              swiftv2_e2e:
+                name: swiftv2_e2e
+                clusterName: "mtcluster"
+              swiftv2_dummy_e2e:
+                name: swiftv2_dummy_e2e
+                clusterName: "swiftv2dummy"              
           steps:
             - template: templates/delete-cluster.yaml
               parameters:
@@ -592,3 +614,35 @@ stages:
                 echo $TAG
                 echo $CURRENT_VERSION
                 echo "Checking if branch is up to date with master"
+
+    - stage: cleanup
+      displayName: Cleanup
+      dependsOn:
+        - azure_overlay_e2e
+        - aks_swift_e2e
+        - cilium_e2e
+        - cilium_overlay_e2e
+        - cilium_h_overlay_e2e
+        - aks_ubuntu_22_linux_e2e
+        - aks_windows_22_e2e
+        - dualstackoverlay_e2e
+        - cilium_dualstackoverlay_e2e
+        - swiftv2_e2e
+      jobs:
+        - job: delete_remote_artifacts
+          displayName: Delete remote artifacts
+          pool:
+            name: $(BUILD_POOL_NAME_DEFAULT)
+            demands: agent.os -equals Linux
+          steps:
+            - checkout: none
+            - task: AzureCLI@1
+              inputs:
+                azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
+                scriptLocation: "inlineScript"
+                inlineScript: |
+                  BUILD_NUMBER=$(Build.BuildNumber)
+                  BUILD_NUMBER=${BUILD_NUMBER//./-}
+                  echo Deleting storage container with name acn-$BUILD_NUMBER and account name $(STORAGE_ACCOUNT_NAME)
+                  az storage container delete -n acn-$BUILD_NUMBER --account-name $(STORAGE_ACCOUNT_NAME)
+              displayName: Cleanup remote Azure storage container

.pipelines/templates/create-cluster-swiftv2.yaml

@@ -0,0 +1,34 @@
+parameters:
+  os: linux
+
+jobs:
+  - job: ${{ parameters.name }}
+    displayName: Cluster - ${{ parameters.name }}
+    steps:
+      - task: AzureCLI@1
+        inputs:
+          azureSubscription: $(ACN_TEST_SERVICE_CONNECTION)
+          scriptLocation: "inlineScript"
+          scriptType: "bash"
+          addSpnToEnvironment: true
+          inlineScript: |
+            set -e
+            echo "Check az version"
+            az version
+            if ${{ lower(contains(parameters.clusterType, 'dualstack')) }}
+            then
+              echo "Install az cli extension preview"
+              az extension add --name aks-preview
+              az extension update --name aks-preview
+            fi
+            mkdir -p ~/.kube/
+            make -C ./hack/aks azcfg AZCLI=az REGION=${{ parameters.region }}
+
+            make -C ./hack/aks ${{ parameters.clusterType }} \
+            AZCLI=az REGION=${{ parameters.region }} SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) \
+            CLUSTER=${{ parameters.clusterName }} \
+            VM_SIZE=${{ parameters.vmSize }} OS=${{parameters.os}} \
+
+            echo "Cluster successfully created"
+        displayName: Cluster - ${{ parameters.clusterType }}
+        continueOnError: ${{ contains(parameters.clusterType, 'dualstack') }}

hack/aks/Makefile

@@ -233,6 +233,28 @@ swift-up: rg-up swift-net-up ## Bring up a SWIFT AzCNI cluster
 		--yes
 	@$(MAKE) set-kubeconf
 
+swiftv2-multitenancy-cluster-up: rg-up
+	$(AZCLI) aks create -n $(CLUSTER) -g $(GROUP) -l $(REGION) \
+		--network-plugin azure \
+		--network-plugin-mode overlay \
+		--kubernetes-version 1.28 \
+		--nodepool-name "mtapool" \
+		--node-vm-size Standard_D4_v2 \
+		--node-count 2 \
+		--nodepool-tags fastpathenabled=true \
+		--no-ssh-key \
+		--yes
+	@$(MAKE) set-kubeconf
+
+swiftv2-dummy-cluster-up: rg-up swift-net-up ## Bring up a SWIFT AzCNI cluster
+	$(AZCLI) aks create -n $(CLUSTER) -g $(GROUP) -l $(REGION) \
+		--network-plugin azure \
+		--vnet-subnet-id /subscriptions/$(SUB)/resourceGroups/$(GROUP)/providers/Microsoft.Network/virtualNetworks/$(VNET)/subnets/nodenet \
+		--pod-subnet-id /subscriptions/$(SUB)/resourceGroups/$(GROUP)/providers/Microsoft.Network/virtualNetworks/$(VNET)/subnets/podnet \
+		--no-ssh-key \
+		--yes
+	@$(MAKE) set-kubeconf
+
 # The below Vnet Scale clusters are currently only in private preview and available with Kubernetes 1.28
 # These AKS clusters can only be created in a limited subscription listed here:
 # https://dev.azure.com/msazure/CloudNativeCompute/_git/aks-rp?path=/resourceprovider/server/microsoft.com/containerservice/flags/network_flags.go&version=GBmaster&line=134&lineEnd=135&lineStartColumn=1&lineEndColumn=1&lineStyle=plain&_a=contents

test/integration/manifests/swiftv2/mtpod0.yaml

@@ -0,0 +1,22 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  labels:
+    kubernetes.azure.com/pod-network: aksswiftvnetv20425
+    kubernetes.azure.com/pod-network-instance: pni1
+  name: mtpod0
+  namespace: default
+spec:
+  containers:
+  - image: nicolaka/netshoot:latest
+    imagePullPolicy: Always
+    name: mtpod0
+    command: ["/bin/bash"]
+    args: ["-c", "while true; do ping localhost; sleep 60;done"]
+    securityContext:
+       privileged: true
+    ports:
+    - containerPort: 80
+      protocol: TCP
+  nodeSelector:
+    kubernetes.io/hostname: $NODE_NAME_0

test/integration/manifests/swiftv2/mtpod1.yaml

@@ -0,0 +1,22 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  labels:
+    kubernetes.azure.com/pod-network: aksswiftvnetv20425
+    kubernetes.azure.com/pod-network-instance: pni1
+  name: mtpod1
+  namespace: default
+spec:
+  containers:
+  - image: nicolaka/netshoot:latest
+    imagePullPolicy: Always
+    name: mtpod1
+    command: ["/bin/bash"]
+    args: ["-c", "while true; do ping localhost; sleep 60;done"]
+    securityContext:
+       privileged: true
+    ports:
+    - containerPort: 80
+      protocol: TCP
+  nodeSelector:
+    kubernetes.io/hostname: $NODE_NAME_1

test/integration/manifests/swiftv2/pni.yaml

@@ -0,0 +1,7 @@
+apiVersion: multitenancy.acn.azure.com/v1alpha1
+kind: PodNetworkInstance
+metadata:
+   name: pni1
+spec:
+   podnetwork: aksswiftvnetv20425
+   podIPReservationSize: 2

test/integration/manifests/swiftv2/podnetwork.yaml

@@ -0,0 +1,10 @@
+apiVersion: multitenancy.acn.azure.com/v1alpha1
+kind: PodNetwork
+metadata:
+  labels:
+    kubernetes.azure.com/override-subnet-token: $SUBNET_TOKEN
+  name: aksswiftvnetv20425
+spec:
+  subnetGUID: $SUBNET_GUID
+  subnetResourceID: $SUBNET_RESOURCE_ID
+  vnetGUID: $VNET_GUID