From dbc8a6b6f3fd50181d8ab7be91c45c5bd05c309a Mon Sep 17 00:00:00 2001 From: Yihong Wang Date: Wed, 22 Mar 2023 13:10:35 -0700 Subject: [PATCH 1/3] increase retry for many edge task Signed-off-by: Yihong Wang --- .tekton/pipeline.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.tekton/pipeline.yaml b/.tekton/pipeline.yaml index 8cc0fb9752..3bf0187e43 100644 --- a/.tekton/pipeline.yaml +++ b/.tekton/pipeline.yaml @@ -659,7 +659,7 @@ spec: - name: task-pvc workspace: pipeline-pvc - name: e2e-test-many-edges - retries: 1 + retries: 2 taskRef: name: e2e-test runAfter: From a96ce68804561400e693c703106affbafec9a019 Mon Sep 17 00:00:00 2001 From: Yihong Wang Date: Mon, 3 Apr 2023 09:44:59 -0700 Subject: [PATCH 2/3] fix(test): update the logic of waiting pods Don't fail the script if there is no pod after the deployment. Instead, do the check again after the sleep. --- scripts/deploy/iks/helper-functions.sh | 36 +++++++++++++------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/scripts/deploy/iks/helper-functions.sh b/scripts/deploy/iks/helper-functions.sh index 84c7dccfbc..68eb374fa0 100755 --- a/scripts/deploy/iks/helper-functions.sh +++ b/scripts/deploy/iks/helper-functions.sh @@ -78,26 +78,26 @@ wait_for_pods () { if [[ -z $pods ]] then - echo "Missing pods." - return 1 - fi - - # Using quotations around variables to keep column format in echo - # Remove 1st line (header line) -> trim whitespace -> cut statuses column (3rd column) - # Might be overkill to parse down to specific columns :). - statuses=$(echo "$pods" | tail -n +2 | tr -s ' ' | cut -d ' ' -f 3) - num_pods=$(echo "$statuses" | wc -l | xargs) - num_running=$(echo "$statuses" | grep -ow "Running\|Completed" | wc -l | xargs) - - local msg="${num_running}/${num_pods} pods running in \"${namespace}\"." - - if [[ $num_running -ne $num_pods ]] - then - echo "$msg Checking again in ${sleep_time}s." + echo "no pod is up yet" else - echo "$msg" - return 0 + # Using quotations around variables to keep column format in echo + # Remove 1st line (header line) -> trim whitespace -> cut statuses column (3rd column) + # Might be overkill to parse down to specific columns :). + statuses=$(echo "$pods" | tail -n +2 | tr -s ' ' | cut -d ' ' -f 3) + num_pods=$(echo "$statuses" | wc -l | xargs) + num_running=$(echo "$statuses" | grep -ow "Running\|Completed" | wc -l | xargs) + + local msg="${num_running}/${num_pods} pods running in \"${namespace}\"." + + if [[ $num_running -ne $num_pods ]] + then + echo "$msg Checking again in ${sleep_time}s." + else + echo "$msg" + return 0 + fi fi + sleep "$sleep_time" i=$((i+1)) done From da9702cb48691fc4e338c01dc26ef59689e226da Mon Sep 17 00:00:00 2001 From: Yihong Wang Date: Mon, 3 Apr 2023 10:59:45 -0700 Subject: [PATCH 3/3] test(fix): randomize the pipeline name use randomized pipeline name to support retry --- scripts/deploy/iks/expect-condition-depend.json | 2 +- scripts/deploy/iks/test-cache.sh | 2 +- scripts/deploy/iks/test-condition-depend.sh | 4 ++-- scripts/deploy/iks/test-flip-coin.sh | 7 ++++--- scripts/deploy/iks/test-many-edges.sh | 7 ++++--- 5 files changed, 12 insertions(+), 10 deletions(-) diff --git a/scripts/deploy/iks/expect-condition-depend.json b/scripts/deploy/iks/expect-condition-depend.json index 631e7a3d6d..c444ba04c3 100644 --- a/scripts/deploy/iks/expect-condition-depend.json +++ b/scripts/deploy/iks/expect-condition-depend.json @@ -2,6 +2,6 @@ "sequence": ["false", "condition-1"], "taskRuns": { "false": { "elapsed": 10}, - "condition-1": {"elapsed": 15} + "condition-1": {"elapsed": 20} } } \ No newline at end of file diff --git a/scripts/deploy/iks/test-cache.sh b/scripts/deploy/iks/test-cache.sh index baa048e260..fbfc7592a7 100755 --- a/scripts/deploy/iks/test-cache.sh +++ b/scripts/deploy/iks/test-cache.sh @@ -21,7 +21,7 @@ run_cache() { shift local PIPELINE_ID local RUN_ID - local PIPELINE_NAME="cache" + local PIPELINE_NAME="cache-$((RANDOM%10000+1))" local KFP_COMMAND="kfp-tekton" echo " ===== cache pipeline =====" diff --git a/scripts/deploy/iks/test-condition-depend.sh b/scripts/deploy/iks/test-condition-depend.sh index ec3bc7331e..b373f98a14 100755 --- a/scripts/deploy/iks/test-condition-depend.sh +++ b/scripts/deploy/iks/test-condition-depend.sh @@ -21,7 +21,7 @@ run_cond_dep() { shift local PIPELINE_ID local RUN_ID - local PIPELINE_NAME="cond-dep" + local PIPELINE_NAME="cond-dep-$((RANDOM%10000+1))" local KFP_COMMAND="kfp-tekton" echo " ===== condition depend pipeline =====" @@ -33,7 +33,7 @@ run_cond_dep() { return "$REV" fi - local RUN_NAME="${PIPELINE_NAME}-run-$((RANDOM%10000+1))" + local RUN_NAME="${PIPELINE_NAME}-run" retry 3 3 $KFP_COMMAND --endpoint http://localhost:8888 run submit -e "exp-cond-dep" -r "$RUN_NAME" -p "$PIPELINE_ID" || : RUN_ID=$($KFP_COMMAND --endpoint http://localhost:8888 run list | grep "$RUN_NAME" | awk '{print $2}') if [[ -z "$RUN_ID" ]]; then diff --git a/scripts/deploy/iks/test-flip-coin.sh b/scripts/deploy/iks/test-flip-coin.sh index 24c6b4f278..2cab114b6c 100755 --- a/scripts/deploy/iks/test-flip-coin.sh +++ b/scripts/deploy/iks/test-flip-coin.sh @@ -22,17 +22,18 @@ run_flip_coin_example() { local PIPELINE_ID local RUN_ID local KFP_COMMAND="kfp-tekton" + local PIPELINE_NAME="e2e-flip-coin-$((RANDOM%10000+1))" echo " ===== flip coin sample =====" python3 samples/flip-coin/condition.py - retry 3 3 $KFP_COMMAND --endpoint http://localhost:8888 pipeline upload -p e2e-flip-coin samples/flip-coin/condition.yaml || : - PIPELINE_ID=$($KFP_COMMAND --endpoint http://localhost:8888 pipeline list | grep 'e2e-flip-coin' | awk '{print $2}') + retry 3 3 $KFP_COMMAND --endpoint http://localhost:8888 pipeline upload -p "$PIPELINE_NAME" samples/flip-coin/condition.yaml || : + PIPELINE_ID=$($KFP_COMMAND --endpoint http://localhost:8888 pipeline list | grep "$PIPELINE_NAME" | awk '{print $2}') if [[ -z "$PIPELINE_ID" ]]; then echo "Failed to upload pipeline" return "$REV" fi - local RUN_NAME="e2e-flip-coin-run-$((RANDOM%10000+1))" + local RUN_NAME="${PIPELINE_NAME}-run" retry 3 3 $KFP_COMMAND --endpoint http://localhost:8888 run submit -e exp-e2e-flip-coin -r "$RUN_NAME" -p "$PIPELINE_ID" || : RUN_ID=$($KFP_COMMAND --endpoint http://localhost:8888 run list | grep "$RUN_NAME" | awk '{print $2}') if [[ -z "$RUN_ID" ]]; then diff --git a/scripts/deploy/iks/test-many-edges.sh b/scripts/deploy/iks/test-many-edges.sh index a08724705c..2bed51df24 100755 --- a/scripts/deploy/iks/test-many-edges.sh +++ b/scripts/deploy/iks/test-many-edges.sh @@ -22,17 +22,18 @@ run_many_edges() { local PIPELINE_ID local RUN_ID local KFP_COMMAND="kfp-tekton" + local PIPELINE_NAME="many-edges-$((RANDOM%10000+1))" echo " ===== many edges =====" python3 scripts/deploy/iks/test/many-edges.py - retry 3 3 $KFP_COMMAND --endpoint http://localhost:8888 pipeline upload -p many-edges scripts/deploy/iks/test/many-edges.yaml || : - PIPELINE_ID=$($KFP_COMMAND --endpoint http://localhost:8888 pipeline list | grep 'many-edges' | awk '{print $2}') + retry 3 3 $KFP_COMMAND --endpoint http://localhost:8888 pipeline upload -p "$PIPELINE_NAME" scripts/deploy/iks/test/many-edges.yaml || : + PIPELINE_ID=$($KFP_COMMAND --endpoint http://localhost:8888 pipeline list | grep "$PIPELINE_NAME" | awk '{print $2}') if [[ -z "$PIPELINE_ID" ]]; then echo "Failed to upload pipeline" return "$REV" fi - local RUN_NAME="many-edges-run-$((RANDOM%10000+1))" + local RUN_NAME="${PIPELINE_NAME}-run" local ENDTIME ENDTIME=$(date -ud "5 second" +%s)