ci: add wait before adding windows nodepool if cluster is in updating state

.pipelines/singletenancy/azure-cni-overlay-stateless/azure-cni-overlay-stateless-e2e-job-template.yaml

@@ -28,31 +28,11 @@ stages:
           k8sVersion: ${{ parameters.k8sVersion }}
           dependsOn: ${{ parameters.dependsOn }}
           region: $(REGION_AKS_CLUSTER_TEST)
-      - job: windows_nodepool
-        displayName: Add Windows Nodepool
-        dependsOn: ${{ parameters.name }}
-        pool:
-          name: $(BUILD_POOL_NAME_DEFAULT)
-          demands:
-          - agent.os -equals Linux
-          - Role -equals $(CUSTOM_E2E_ROLE)
-        steps:
-          - task: AzureCLI@2
-            inputs:
-              azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
-              scriptLocation: "inlineScript"
-              scriptType: "bash"
-              addSpnToEnvironment: true
-              inlineScript: |
-                set -e
-                make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(commitID)
-                make -C ./hack/aks windows-nodepool-up AZCLI=az SUB=$(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS) CLUSTER=${{ parameters.clusterName }}-$(commitID) VM_SIZE_WIN=${{ parameters.vmSize }}
-                echo "Windows node are successfully added to v4 Overlay Cluster"
-                kubectl cluster-info
-                kubectl get node -owide
-                kubectl get po -owide -A
-            name: "Add_Windows_Node"
-            displayName: "Add windows node on v4 overlay cluster"
+      - template: ../../templates/add-windows-nodepool-job.yaml
+        parameters:
+          depend: ${{ parameters.name }}
+          clusterName: ${{ parameters.clusterName }}-$(commitID)
+          vmSize: ${{ parameters.vmSize }}
 
   - stage: ${{ parameters.name }}
     displayName: E2E - ${{ parameters.displayName }}

.pipelines/singletenancy/azure-cni-overlay/azure-cni-overlay-e2e-job-template.yaml

@@ -58,34 +58,11 @@ stages:
               clusterName: ${{ parameters.clusterName }}-$(commitID)
               os: linux
               scaleup: 100
-
-      - job: windows_nodepool
-        displayName: Add Windows Nodepool
-        dependsOn: ${{ parameters.name }}_linux
-        pool:
-          name: $(BUILD_POOL_NAME_DEFAULT)
-          demands:
-          - agent.os -equals Linux
-          - Role -equals $(CUSTOM_E2E_ROLE)
-        steps:
-          - task: AzureCLI@2
-            inputs:
-              azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
-              scriptLocation: "inlineScript"
-              scriptType: "bash"
-              addSpnToEnvironment: true
-              inlineScript: |
-                set -e
-                make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(commitID)
-                make -C ./hack/aks windows-nodepool-up AZCLI=az SUB=$(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS) CLUSTER=${{ parameters.clusterName }}-$(commitID) VM_SIZE_WIN=${{ parameters.vmSize }}
-                echo "Windows node are successfully added to v4 Overlay Cluster"
-                kubectl cluster-info
-                kubectl get node -owide
-                kubectl get po -owide -A
-            name: "Add_Windows_Node"
-            displayName: "Add windows node on v4 overlay cluster"
-
-
+      - template: ../../templates/add-windows-nodepool-job.yaml
+        parameters:
+          depend: ${{ parameters.name }}_linux
+          clusterName: ${{ parameters.clusterName }}-$(commitID)
+          vmSize: ${{ parameters.vmSize }}
       - job: ${{ parameters.name }}_windows
         displayName: Azure CNI Overlay Test Suite | Windows - (${{ parameters.name }})
         timeoutInMinutes: 120

.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-job-template.yaml

@@ -59,33 +59,11 @@ stages:
               clusterName: ${{ parameters.clusterName }}-$(commitID)
               os: linux
               scaleup: 100
-
-      - job: windows_nodepool
-        displayName: Add Windows Nodepool
-        dependsOn: ${{ parameters.name }}_linux
-        pool:
-          name: $(BUILD_POOL_NAME_DEFAULT)
-          demands:
-          - agent.os -equals Linux
-          - Role -equals $(CUSTOM_E2E_ROLE)
-        steps:
-          - task: AzureCLI@2
-            inputs:
-              azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
-              scriptLocation: "inlineScript"
-              scriptType: "bash"
-              addSpnToEnvironment: true
-              inlineScript: |
-                set -e
-                make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(commitID)
-                make -C ./hack/aks windows-nodepool-up AZCLI=az SUB=$(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS) CLUSTER=${{ parameters.clusterName }}-$(commitID) VM_SIZE_WIN=${{ parameters.vmSize }}
-                echo "Windows nodes have been successfully added to DualStack Overlay Cluster"
-                kubectl cluster-info
-                kubectl get node -owide
-                kubectl get po -owide -A
-            name: "Add_Windows_Node"
-            displayName: "Add windows node"
-
+      - template: ../../templates/add-windows-nodepool-job.yaml
+        parameters:
+          depend: ${{ parameters.name }}_linux
+          clusterName: ${{ parameters.clusterName }}-$(commitID)
+          vmSize: "Standard_D2_v3"
       - job: ${{ parameters.name }}_windows
         displayName: DualStack Overlay Test Suite | Windows - (${{ parameters.name }})
         timeoutInMinutes: 120

.pipelines/templates/add-windows-nodepool-job.yaml

@@ -0,0 +1,63 @@
+parameters:
+  depend: ""
+  clusterName: "" # unique identifier
+  vmSize: ""
+
+jobs:
+- job: windows_nodepool
+  displayName: Add Windows Nodepool
+  dependsOn: ${{ parameters.depend }}
+  pool:
+    name: $(BUILD_POOL_NAME_DEFAULT)
+    demands:
+    - agent.os -equals Linux
+    - Role -equals $(CUSTOM_E2E_ROLE)
+  timeoutInMinutes: 30
+  steps:
+    - task: AzureCLI@2
+      inputs:
+        azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
+        scriptLocation: "inlineScript"
+        scriptType: "bash"
+        addSpnToEnvironment: true
+        inlineScript: |
+          set -e
+
+          windows_nodepool=$(az aks nodepool list \
+          --resource-group ${{ parameters.clusterName }} \
+          --cluster-name ${{ parameters.clusterName }} \
+          --query "[?osType=='Windows']" \
+          --output tsv)
+
+          if [ -z "$windows_nodepool" ]; then
+            echo "No Windows node pool found in the AKS cluster."
+
+            # wait for cluster to update
+            while true; do
+              cluster_state=$(az aks show \
+              --name "${{ parameters.clusterName }}" \
+              --resource-group "${{ parameters.clusterName }}" \
+              --query provisioningState)
+
+              if echo "$cluster_state" | grep -q "Updating"; then
+                echo "Cluster is updating. Sleeping for 30 seconds..."
+                sleep 30
+              else
+                break
+              fi
+            done
+            # cluster state is always set and visible outside the loop
+            echo "Cluster state is: $cluster_state"
+
+            make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}
+            make -C ./hack/aks windows-nodepool-up AZCLI=az SUB=$(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS) CLUSTER=${{ parameters.clusterName }} VM_SIZE_WIN=${{ parameters.vmSize }}
+            echo "Windows node was successfully added"
+            kubectl cluster-info
+            kubectl get node -owide
+            kubectl get po -owide -A
+          else
+            echo "Windows node pool already exists in the AKS cluster."
+          fi
+      name: "Add_Windows_Node"
+      displayName: "Add windows node to cluster"
+      retryCountOnTaskFailure: 5