ci: fix cilium validate state check for PRs (#1962)

* updating validate state check * fix cluster create * remove makefile changes
Azure · May 17, 2023 · 47a358c · 47a358c
1 parent d711636
commit 47a358c
Show file tree

Hide file tree

Showing 3 changed files with 27 additions and 17 deletions.
diff --git a/.pipelines/singletenancy/cilium/cilium-e2e-step-template.yaml b/.pipelines/singletenancy/cilium/cilium-e2e-step-template.yaml
@@ -107,14 +107,6 @@ steps:
       pathtoPublish: "$(Build.ArtifactStagingDirectory)/test-output"
     condition: always()
 
-  - script: |
-      echo "validate pod IP assignment and check systemd-networkd restart"
-      kubectl apply -f hack/manifests/hostprocess.yaml
-      kubectl get pod -owide -A
-      bash hack/scripts/validate_state.sh
-    name: "validatePods"
-    displayName: "Validate Pods"
-
   - script: |
       echo "Run Service Conformance E2E"
       export PATH=${PATH}:/usr/local/bin/gsutil
@@ -131,6 +123,18 @@ steps:
     retryCountOnTaskFailure: 3
     name: "ciliumConnectivityTests"
     displayName: "Run Cilium Connectivity Tests"
+
+  - script: |
+      echo "validate pod IP assignment and check systemd-networkd restart"
+      kubectl apply -f hack/manifests/hostprocess.yaml
+      kubectl get pod -owide -A
+      bash hack/scripts/validate_state.sh
+      echo "delete cilium connectivity test resources and re-validate state"
+      kubectl delete ns cilium-test
+      kubectl get pod -owide -A
+      bash hack/scripts/validate_state.sh
+    name: "validatePods"
+    displayName: "Validate Pods"
   
   - script: |
       ARTIFACT_DIR=$(Build.ArtifactStagingDirectory)/test-output/

diff --git a/.pipelines/singletenancy/overlay/overlay-e2e-step-template.yaml b/.pipelines/singletenancy/overlay/overlay-e2e-step-template.yaml
@@ -113,14 +113,6 @@ steps:
       pathtoPublish: "$(Build.ArtifactStagingDirectory)/test-output"
     condition: always()
 
-  - script: |
-      echo "validate pod IP assignment and check systemd-networkd restart"
-      kubectl apply -f hack/manifests/hostprocess.yaml
-      kubectl get pod -owide -A
-      bash hack/scripts/validate_state.sh
-    name: "validatePods"
-    displayName: "Validate Pods"
-
   - script: |
       echo "Run Service Conformance E2E"
       export PATH=${PATH}:/usr/local/bin/gsutil
@@ -136,6 +128,18 @@ steps:
       cilium connectivity test
     name: "ciliumConnectivityTests"
     displayName: "Run Cilium Connectivity Tests"
+
+  - script: |
+      echo "validate pod IP assignment and check systemd-networkd restart"
+      kubectl apply -f hack/manifests/hostprocess.yaml
+      kubectl get pod -owide -A
+      bash hack/scripts/validate_state.sh
+      echo "delete cilium connectivity test resources and re-validate state"
+      kubectl delete ns cilium-test
+      kubectl get pod -owide -A
+      bash hack/scripts/validate_state.sh
+    name: "validatePods"
+    displayName: "Validate Pods"
   
   - script: |
       ARTIFACT_DIR=$(Build.ArtifactStagingDirectory)/test-output/

diff --git a/hack/scripts/validate_state.sh b/hack/scripts/validate_state.sh
@@ -15,9 +15,11 @@ do
     node_name="${node##*/}"
     node_ip=$(kubectl get "$node"  -o jsonpath='{$.status.addresses[?(@.type=="InternalIP")].address}')
     echo "Node internal ip: $node_ip"
+    # Check pod count after restarting nodes, statefile does not exist after restart
     echo "checking whether the node has any pods deployed to it or not"
-    pod_count=$(kubectl get pods -o wide | grep "$node_name" -c)
+    pod_count=$(kubectl get pods -A -o wide | grep "$node_name" -c)
     if [[ $pod_count -eq 0 ]]; then
+        echo "Skipping validation for this node. No pods were deployed after the restart, so no statefile exists"
         continue
     fi
     privileged_pod=$(kubectl get pods -n kube-system -l app=privileged-daemonset -o wide | grep "$node_name" | awk '{print $1}')