[testing] Fix e2e tests timed out, but there's empty test log (#2810)

* Wait longer for e2e tests * Let api integration test stream log output while testing * Also fix initialization test * Fix exit code on error * Update test/initialization-test/run_test.sh Co-Authored-By: Alexey Volkov <[email protected]> * Update according to CR Co-authored-by: Alexey Volkov <[email protected]>
kubeflow · Jan 10, 2020 · 493c3d4 · 493c3d4
1 parent 10a4f55
commit 493c3d4
Show file tree

Hide file tree

Showing 3 changed files with 13 additions and 11 deletions.
diff --git a/test/api-integration-test/run_test.sh b/test/api-integration-test/run_test.sh
@@ -15,6 +15,7 @@
 # limitations under the License.
 
 set -xe
+set -o pipefail
 
 # K8s Namespace that all resources deployed to
 NAMESPACE=kubeflow
@@ -64,13 +65,13 @@ cd "${BASE_DIR}/${TEST_DIR}"
 export GO111MODULE=on
 
 echo "Run integration test..."
-TEST_RESULT=`go test -v ./... -namespace ${NAMESPACE} -args -runIntegrationTests=true 2>&1`
-TEST_EXIT_CODE=$?
+LOG_FILE=$(mktemp)
+# Note, "set -o pipefail" at top of file is required to catch exit code of the pipe.
+TEST_EXIT_CODE=0 # reference for how to save exit code: https://stackoverflow.com/a/18622662
+go test -v ./... -namespace ${NAMESPACE} -args -runIntegrationTests=true |& tee $LOG_FILE || TEST_EXIT_CODE=$?
 
-# Log the test result
-printf '%s\n' "$TEST_RESULT"
 # Convert test result to junit.xml
-printf '%s\n' "$TEST_RESULT" | go-junit-report > ${JUNIT_TEST_RESULT}
+< "$LOG_FILE" go-junit-report > "${JUNIT_TEST_RESULT}"
 
 echo "Copy test result to GCS ${RESULTS_GCS_DIR}/${JUNIT_TEST_RESULT}"
 gsutil cp ${JUNIT_TEST_RESULT} ${RESULTS_GCS_DIR}/${JUNIT_TEST_RESULT}

diff --git a/test/initialization-test/run_test.sh b/test/initialization-test/run_test.sh
@@ -15,6 +15,7 @@
 # limitations under the License.
 
 set -xe
+set -o pipefail
 
 # K8s Namespace that all resources deployed to
 NAMESPACE=kubeflow
@@ -64,13 +65,13 @@ cd "${BASE_DIR}/${TEST_DIR}"
 export GO111MODULE=on
 
 echo "Run Initialization test..."
-TEST_RESULT=`go test -v ./... -namespace ${NAMESPACE} -args -runIntegrationTests=true 2>&1`
-TEST_EXIT_CODE=$?
+LOG_FILE=$(mktemp)
+# Note, "set -o pipefail" at top of file is required to catch exit code of the pipe.
+TEST_EXIT_CODE=0 # reference for how to save exit code: https://stackoverflow.com/a/18622662
+go test -v ./... -namespace ${NAMESPACE} -args -runIntegrationTests=true |& tee $LOG_FILE || TEST_EXIT_CODE=$?
 
-# Log the test result
-printf '%s\n' "$TEST_RESULT"
 # Convert test result to junit.xml
-printf '%s\n' "$TEST_RESULT" | go-junit-report > ${JUNIT_TEST_RESULT}
+< "$LOG_FILE" go-junit-report > "${JUNIT_TEST_RESULT}"
 
 echo "Copy test result to GCS ${RESULTS_GCS_DIR}/${JUNIT_TEST_RESULT}"
 gsutil cp ${JUNIT_TEST_RESULT} ${RESULTS_GCS_DIR}/${JUNIT_TEST_RESULT}

diff --git a/test/presubmit-tests-with-pipeline-deployment.sh b/test/presubmit-tests-with-pipeline-deployment.sh
@@ -31,7 +31,7 @@ usage()
 PLATFORM=gcp
 PROJECT=ml-pipeline-test
 TEST_RESULT_BUCKET=ml-pipeline-test
-TIMEOUT_SECONDS=1800
+TIMEOUT_SECONDS=1800 # 30 minutes
 NAMESPACE=kubeflow
 ENABLE_WORKLOAD_IDENTITY=true