From 2268eaebce4344ab4d03a6b2e8fb84deeb0167c7 Mon Sep 17 00:00:00 2001 From: "Yuan (Bob) Gong" Date: Tue, 7 Jan 2020 17:20:31 +0800 Subject: [PATCH 1/6] Wait longer for e2e tests --- test/presubmit-tests-with-pipeline-deployment.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/presubmit-tests-with-pipeline-deployment.sh b/test/presubmit-tests-with-pipeline-deployment.sh index d3602046fea..8011c0baef3 100755 --- a/test/presubmit-tests-with-pipeline-deployment.sh +++ b/test/presubmit-tests-with-pipeline-deployment.sh @@ -31,7 +31,7 @@ usage() PLATFORM=gcp PROJECT=ml-pipeline-test TEST_RESULT_BUCKET=ml-pipeline-test -TIMEOUT_SECONDS=1800 +TIMEOUT_SECONDS=3600 # 1 hour NAMESPACE=kubeflow ENABLE_WORKLOAD_IDENTITY=true From a5ef4b254225e0ce0d56f1b4c3170e273a5e95ce Mon Sep 17 00:00:00 2001 From: Yuan Gong Date: Tue, 7 Jan 2020 17:57:05 +0800 Subject: [PATCH 2/6] Let api integration test stream log output while testing --- test/api-integration-test/run_test.sh | 9 ++++----- test/presubmit-tests-with-pipeline-deployment.sh | 2 +- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/test/api-integration-test/run_test.sh b/test/api-integration-test/run_test.sh index 915a6d2e1b8..8ee4cfe4f02 100755 --- a/test/api-integration-test/run_test.sh +++ b/test/api-integration-test/run_test.sh @@ -64,13 +64,12 @@ cd "${BASE_DIR}/${TEST_DIR}" export GO111MODULE=on echo "Run integration test..." -TEST_RESULT=`go test -v ./... -namespace ${NAMESPACE} -args -runIntegrationTests=true 2>&1` -TEST_EXIT_CODE=$? +LOG_FILE=$(mktemp) +TEST_EXIT_CODE=0 # reference for how to save exit code: https://stackoverflow.com/a/18622662 +go test -v ./... -namespace ${NAMESPACE} -args -runIntegrationTests=true |& tee $LOG_FILE || TEST_EXIT_CODE=$? -# Log the test result -printf '%s\n' "$TEST_RESULT" # Convert test result to junit.xml -printf '%s\n' "$TEST_RESULT" | go-junit-report > ${JUNIT_TEST_RESULT} +cat $LOG_FILE | go-junit-report > ${JUNIT_TEST_RESULT} echo "Copy test result to GCS ${RESULTS_GCS_DIR}/${JUNIT_TEST_RESULT}" gsutil cp ${JUNIT_TEST_RESULT} ${RESULTS_GCS_DIR}/${JUNIT_TEST_RESULT} diff --git a/test/presubmit-tests-with-pipeline-deployment.sh b/test/presubmit-tests-with-pipeline-deployment.sh index 8011c0baef3..2ba7b521f59 100755 --- a/test/presubmit-tests-with-pipeline-deployment.sh +++ b/test/presubmit-tests-with-pipeline-deployment.sh @@ -31,7 +31,7 @@ usage() PLATFORM=gcp PROJECT=ml-pipeline-test TEST_RESULT_BUCKET=ml-pipeline-test -TIMEOUT_SECONDS=3600 # 1 hour +TIMEOUT_SECONDS=1800 # 30 minutes NAMESPACE=kubeflow ENABLE_WORKLOAD_IDENTITY=true From 819a54dee1bf503c87ad31a86e602c9faa4ce09a Mon Sep 17 00:00:00 2001 From: Yuan Gong Date: Tue, 7 Jan 2020 18:05:22 +0800 Subject: [PATCH 3/6] Also fix initialization test --- test/initialization-test/run_test.sh | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/test/initialization-test/run_test.sh b/test/initialization-test/run_test.sh index 49d553e299b..6ef17b45ed4 100755 --- a/test/initialization-test/run_test.sh +++ b/test/initialization-test/run_test.sh @@ -64,13 +64,12 @@ cd "${BASE_DIR}/${TEST_DIR}" export GO111MODULE=on echo "Run Initialization test..." -TEST_RESULT=`go test -v ./... -namespace ${NAMESPACE} -args -runIntegrationTests=true 2>&1` -TEST_EXIT_CODE=$? +LOG_FILE=$(mktemp) +TEST_EXIT_CODE=0 # reference for how to save exit code: https://stackoverflow.com/a/18622662 +go test -v ./... -namespace ${NAMESPACE} -args -runIntegrationTests=true |& tee $LOG_FILE || TEST_EXIT_CODE=$? -# Log the test result -printf '%s\n' "$TEST_RESULT" # Convert test result to junit.xml -printf '%s\n' "$TEST_RESULT" | go-junit-report > ${JUNIT_TEST_RESULT} +cat $LOG_FILE | go-junit-report > ${JUNIT_TEST_RESULT} echo "Copy test result to GCS ${RESULTS_GCS_DIR}/${JUNIT_TEST_RESULT}" gsutil cp ${JUNIT_TEST_RESULT} ${RESULTS_GCS_DIR}/${JUNIT_TEST_RESULT} From d7c5dba1105c8da60cf34caa71bc87875f1c03bc Mon Sep 17 00:00:00 2001 From: Yuan Gong Date: Thu, 9 Jan 2020 15:12:19 +0800 Subject: [PATCH 4/6] Fix exit code on error --- test/api-integration-test/run_test.sh | 2 ++ test/initialization-test/run_test.sh | 2 ++ 2 files changed, 4 insertions(+) diff --git a/test/api-integration-test/run_test.sh b/test/api-integration-test/run_test.sh index 8ee4cfe4f02..ed45ea986c8 100755 --- a/test/api-integration-test/run_test.sh +++ b/test/api-integration-test/run_test.sh @@ -15,6 +15,7 @@ # limitations under the License. set -xe +set -o pipefail # K8s Namespace that all resources deployed to NAMESPACE=kubeflow @@ -65,6 +66,7 @@ export GO111MODULE=on echo "Run integration test..." LOG_FILE=$(mktemp) +# Note, "set -o pipefail" at top of file is required to catch exit code of the pipe. TEST_EXIT_CODE=0 # reference for how to save exit code: https://stackoverflow.com/a/18622662 go test -v ./... -namespace ${NAMESPACE} -args -runIntegrationTests=true |& tee $LOG_FILE || TEST_EXIT_CODE=$? diff --git a/test/initialization-test/run_test.sh b/test/initialization-test/run_test.sh index 6ef17b45ed4..1c0e95c79ca 100755 --- a/test/initialization-test/run_test.sh +++ b/test/initialization-test/run_test.sh @@ -15,6 +15,7 @@ # limitations under the License. set -xe +set -o pipefail # K8s Namespace that all resources deployed to NAMESPACE=kubeflow @@ -65,6 +66,7 @@ export GO111MODULE=on echo "Run Initialization test..." LOG_FILE=$(mktemp) +# Note, "set -o pipefail" at top of file is required to catch exit code of the pipe. TEST_EXIT_CODE=0 # reference for how to save exit code: https://stackoverflow.com/a/18622662 go test -v ./... -namespace ${NAMESPACE} -args -runIntegrationTests=true |& tee $LOG_FILE || TEST_EXIT_CODE=$? From e3c96b6b6a88295be1256e933931df14a4b42a83 Mon Sep 17 00:00:00 2001 From: "Yuan (Bob) Gong" Date: Thu, 9 Jan 2020 15:16:26 +0800 Subject: [PATCH 5/6] Update test/initialization-test/run_test.sh Co-Authored-By: Alexey Volkov --- test/initialization-test/run_test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/initialization-test/run_test.sh b/test/initialization-test/run_test.sh index 1c0e95c79ca..f7c97790a38 100755 --- a/test/initialization-test/run_test.sh +++ b/test/initialization-test/run_test.sh @@ -71,7 +71,7 @@ TEST_EXIT_CODE=0 # reference for how to save exit code: https://stackoverflow.co go test -v ./... -namespace ${NAMESPACE} -args -runIntegrationTests=true |& tee $LOG_FILE || TEST_EXIT_CODE=$? # Convert test result to junit.xml -cat $LOG_FILE | go-junit-report > ${JUNIT_TEST_RESULT} +< "$LOG_FILE" go-junit-report > "${JUNIT_TEST_RESULT}" echo "Copy test result to GCS ${RESULTS_GCS_DIR}/${JUNIT_TEST_RESULT}" gsutil cp ${JUNIT_TEST_RESULT} ${RESULTS_GCS_DIR}/${JUNIT_TEST_RESULT} From 84d491c1c136bf7e67dbee4cad2d2d7078a3a09e Mon Sep 17 00:00:00 2001 From: Yuan Gong Date: Thu, 9 Jan 2020 15:18:55 +0800 Subject: [PATCH 6/6] Update according to CR --- test/api-integration-test/run_test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/api-integration-test/run_test.sh b/test/api-integration-test/run_test.sh index ed45ea986c8..fc4b4298a4c 100755 --- a/test/api-integration-test/run_test.sh +++ b/test/api-integration-test/run_test.sh @@ -71,7 +71,7 @@ TEST_EXIT_CODE=0 # reference for how to save exit code: https://stackoverflow.co go test -v ./... -namespace ${NAMESPACE} -args -runIntegrationTests=true |& tee $LOG_FILE || TEST_EXIT_CODE=$? # Convert test result to junit.xml -cat $LOG_FILE | go-junit-report > ${JUNIT_TEST_RESULT} +< "$LOG_FILE" go-junit-report > "${JUNIT_TEST_RESULT}" echo "Copy test result to GCS ${RESULTS_GCS_DIR}/${JUNIT_TEST_RESULT}" gsutil cp ${JUNIT_TEST_RESULT} ${RESULTS_GCS_DIR}/${JUNIT_TEST_RESULT}