fix(test): enhance build scripts (#1200)

* increase retry for many edge task Signed-off-by: Yihong Wang <[email protected]> * fix(test): update the logic of waiting pods Don't fail the script if there is no pod after the deployment. Instead, do the check again after the sleep. * test(fix): randomize the pipeline name use randomized pipeline name to support retry --------- Signed-off-by: Yihong Wang <[email protected]>
kubeflow · Apr 3, 2023 · 4764448 · 4764448
1 parent 1f24ba5
commit 4764448
Show file tree

Hide file tree

Showing 7 changed files with 31 additions and 29 deletions.
diff --git a/.tekton/pipeline.yaml b/.tekton/pipeline.yaml
@@ -659,7 +659,7 @@ spec:
       - name: task-pvc
         workspace: pipeline-pvc
     - name: e2e-test-many-edges
-      retries: 1
+      retries: 2
       taskRef:
         name: e2e-test
       runAfter:

diff --git a/scripts/deploy/iks/expect-condition-depend.json b/scripts/deploy/iks/expect-condition-depend.json
@@ -2,6 +2,6 @@
     "sequence": ["false", "condition-1"],
     "taskRuns": {
         "false": { "elapsed": 10},
-        "condition-1": {"elapsed": 15}
+        "condition-1": {"elapsed": 20}
     }
 }
diff --git a/scripts/deploy/iks/helper-functions.sh b/scripts/deploy/iks/helper-functions.sh
@@ -78,26 +78,26 @@ wait_for_pods () {
 
         if [[ -z $pods ]]
         then
-            echo "Missing pods."
-            return 1
-        fi
-
-        # Using quotations around variables to keep column format in echo
-        # Remove 1st line (header line) -> trim whitespace -> cut statuses column (3rd column)
-        # Might be overkill to parse down to specific columns :).
-        statuses=$(echo "$pods" | tail -n +2 | tr -s ' '  | cut -d ' ' -f 3)
-        num_pods=$(echo "$statuses" | wc -l | xargs)
-        num_running=$(echo "$statuses" | grep -ow "Running\|Completed" | wc -l | xargs)
-
-        local msg="${num_running}/${num_pods} pods running in \"${namespace}\"."
-
-        if [[ $num_running -ne $num_pods ]]
-        then
-            echo "$msg Checking again in ${sleep_time}s."
+            echo "no pod is up yet"
         else
-            echo "$msg"
-            return 0
+            # Using quotations around variables to keep column format in echo
+            # Remove 1st line (header line) -> trim whitespace -> cut statuses column (3rd column)
+            # Might be overkill to parse down to specific columns :).
+            statuses=$(echo "$pods" | tail -n +2 | tr -s ' '  | cut -d ' ' -f 3)
+            num_pods=$(echo "$statuses" | wc -l | xargs)
+            num_running=$(echo "$statuses" | grep -ow "Running\|Completed" | wc -l | xargs)
+
+            local msg="${num_running}/${num_pods} pods running in \"${namespace}\"."
+
+            if [[ $num_running -ne $num_pods ]]
+            then
+                echo "$msg Checking again in ${sleep_time}s."
+            else
+                echo "$msg"
+                return 0
+            fi
         fi
+
         sleep "$sleep_time"
         i=$((i+1))
     done

diff --git a/scripts/deploy/iks/test-cache.sh b/scripts/deploy/iks/test-cache.sh
@@ -21,7 +21,7 @@ run_cache() {
   shift
   local PIPELINE_ID
   local RUN_ID
-  local PIPELINE_NAME="cache"
+  local PIPELINE_NAME="cache-$((RANDOM%10000+1))"
   local KFP_COMMAND="kfp-tekton"
 
   echo " =====   cache pipeline  ====="

diff --git a/scripts/deploy/iks/test-condition-depend.sh b/scripts/deploy/iks/test-condition-depend.sh
@@ -21,7 +21,7 @@ run_cond_dep() {
   shift
   local PIPELINE_ID
   local RUN_ID
-  local PIPELINE_NAME="cond-dep"
+  local PIPELINE_NAME="cond-dep-$((RANDOM%10000+1))"
   local KFP_COMMAND="kfp-tekton"
 
   echo " =====   condition depend pipeline  ====="
@@ -33,7 +33,7 @@ run_cond_dep() {
     return "$REV"
   fi
 
-  local RUN_NAME="${PIPELINE_NAME}-run-$((RANDOM%10000+1))"
+  local RUN_NAME="${PIPELINE_NAME}-run"
   retry 3 3 $KFP_COMMAND --endpoint http://localhost:8888 run submit -e "exp-cond-dep" -r "$RUN_NAME" -p "$PIPELINE_ID" || :
   RUN_ID=$($KFP_COMMAND --endpoint http://localhost:8888  run list | grep "$RUN_NAME" | awk '{print $2}')
   if [[ -z "$RUN_ID" ]]; then

diff --git a/scripts/deploy/iks/test-flip-coin.sh b/scripts/deploy/iks/test-flip-coin.sh
@@ -22,17 +22,18 @@ run_flip_coin_example() {
   local PIPELINE_ID
   local RUN_ID
   local KFP_COMMAND="kfp-tekton"
+  local PIPELINE_NAME="e2e-flip-coin-$((RANDOM%10000+1))"
 
   echo " =====   flip coin sample  ====="
   python3 samples/flip-coin/condition.py
-  retry 3 3 $KFP_COMMAND --endpoint http://localhost:8888 pipeline upload -p e2e-flip-coin samples/flip-coin/condition.yaml || :
-  PIPELINE_ID=$($KFP_COMMAND --endpoint http://localhost:8888  pipeline list | grep 'e2e-flip-coin' | awk '{print $2}')
+  retry 3 3 $KFP_COMMAND --endpoint http://localhost:8888 pipeline upload -p "$PIPELINE_NAME" samples/flip-coin/condition.yaml || :
+  PIPELINE_ID=$($KFP_COMMAND --endpoint http://localhost:8888  pipeline list | grep "$PIPELINE_NAME" | awk '{print $2}')
   if [[ -z "$PIPELINE_ID" ]]; then
     echo "Failed to upload pipeline"
     return "$REV"
   fi
 
-  local RUN_NAME="e2e-flip-coin-run-$((RANDOM%10000+1))"
+  local RUN_NAME="${PIPELINE_NAME}-run"
   retry 3 3 $KFP_COMMAND --endpoint http://localhost:8888 run submit -e exp-e2e-flip-coin -r "$RUN_NAME" -p "$PIPELINE_ID" || :
   RUN_ID=$($KFP_COMMAND --endpoint http://localhost:8888  run list | grep "$RUN_NAME" | awk '{print $2}')
   if [[ -z "$RUN_ID" ]]; then

diff --git a/scripts/deploy/iks/test-many-edges.sh b/scripts/deploy/iks/test-many-edges.sh
@@ -22,17 +22,18 @@ run_many_edges() {
   local PIPELINE_ID
   local RUN_ID
   local KFP_COMMAND="kfp-tekton"
+  local PIPELINE_NAME="many-edges-$((RANDOM%10000+1))"
 
   echo " =====  many edges  ====="
   python3 scripts/deploy/iks/test/many-edges.py
-  retry 3 3 $KFP_COMMAND --endpoint http://localhost:8888 pipeline upload -p many-edges scripts/deploy/iks/test/many-edges.yaml || :
-  PIPELINE_ID=$($KFP_COMMAND --endpoint http://localhost:8888  pipeline list | grep 'many-edges' | awk '{print $2}')
+  retry 3 3 $KFP_COMMAND --endpoint http://localhost:8888 pipeline upload -p "$PIPELINE_NAME" scripts/deploy/iks/test/many-edges.yaml || :
+  PIPELINE_ID=$($KFP_COMMAND --endpoint http://localhost:8888  pipeline list | grep "$PIPELINE_NAME" | awk '{print $2}')
   if [[ -z "$PIPELINE_ID" ]]; then
     echo "Failed to upload pipeline"
     return "$REV"
   fi
 
-  local RUN_NAME="many-edges-run-$((RANDOM%10000+1))"
+  local RUN_NAME="${PIPELINE_NAME}-run"
   local ENDTIME
 
   ENDTIME=$(date -ud "5 second" +%s)