diff --git a/.github/workflows/codeql_analysis.yml b/.github/workflows/codeql_analysis.yml index ad61fd132d9..bd710c71cd8 100644 --- a/.github/workflows/codeql_analysis.yml +++ b/.github/workflows/codeql_analysis.yml @@ -63,12 +63,9 @@ jobs: # Install everything else we need, and configure sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget - + sudo service etcd stop - - # install JUnit report formatter - go install github.com/vitessio/go-junit-report@HEAD - + - name: Building binaries timeout-minutes: 30 run: | diff --git a/.github/workflows/unit_race.yml b/.github/workflows/unit_race.yml index 0ca3f6f8d4c..12c0d47459f 100644 --- a/.github/workflows/unit_race.yml +++ b/.github/workflows/unit_race.yml @@ -91,9 +91,6 @@ jobs: go mod download go install golang.org/x/tools/cmd/goimports@latest - # install JUnit report formatter - go install github.com/vitessio/go-junit-report@HEAD - - name: Run make tools if: steps.changes.outputs.unit_tests == 'true' run: | @@ -127,7 +124,7 @@ jobs: # testing, e.g. MySQL 5.7 vs 8.0. export CI_DB_PLATFORM="mysql80" - make unit_test_race | tee -a output.txt | go-junit-report -set-exit-code > report.xml + JUNIT_OUTPUT=report.xml JSON_OUTPUT=report.json make unit_test_race - name: Record test results in launchable if PR is not a draft if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.changes.outputs.unit_tests == 'true' && github.base_ref == 'main' && !cancelled() @@ -135,15 +132,17 @@ jobs: # send recorded tests to launchable launchable record tests --build "$GITHUB_RUN_ID" go-test . || true - - name: Print test output - if: steps.changes.outputs.unit_tests == 'true' && !cancelled() - run: | - # print test output - cat output.txt - - name: Test Summary if: steps.changes.outputs.unit_tests == 'true' && !cancelled() uses: test-summary/action@31493c76ec9e7aa675f1585d3ed6f1da69269a86 # v2.4 with: paths: "report.xml" show: "fail" + + - name: Slowest Tests + if: steps.changes.outputs.unit_tests == 'true' && !cancelled() + run: | + echo '## Slowest Tests' >> "$GITHUB_STEP_SUMMARY" + echo '```' >> "$GITHUB_STEP_SUMMARY" + go tool gotestsum tool slowest --num 20 --jsonfile report.json | tee -a "$GITHUB_STEP_SUMMARY" + echo '```' >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/unit_race_evalengine.yml b/.github/workflows/unit_race_evalengine.yml index b88e5147088..f9553200f0a 100644 --- a/.github/workflows/unit_race_evalengine.yml +++ b/.github/workflows/unit_race_evalengine.yml @@ -91,9 +91,6 @@ jobs: go mod download go install golang.org/x/tools/cmd/goimports@latest - # install JUnit report formatter - go install github.com/vitessio/go-junit-report@HEAD - - name: Run make tools if: steps.changes.outputs.unit_tests == 'true' run: | @@ -127,7 +124,7 @@ jobs: # testing, e.g. MySQL 5.7 vs 8.0. export CI_DB_PLATFORM="mysql80" - make unit_test_race | tee -a output.txt | go-junit-report -set-exit-code > report.xml + JUNIT_OUTPUT=report.xml JSON_OUTPUT=report.json make unit_test_race - name: Record test results in launchable if PR is not a draft if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.changes.outputs.unit_tests == 'true' && github.base_ref == 'main' && !cancelled() @@ -135,15 +132,17 @@ jobs: # send recorded tests to launchable launchable record tests --build "$GITHUB_RUN_ID" go-test . || true - - name: Print test output - if: steps.changes.outputs.unit_tests == 'true' && !cancelled() - run: | - # print test output - cat output.txt - - name: Test Summary if: steps.changes.outputs.unit_tests == 'true' && !cancelled() uses: test-summary/action@31493c76ec9e7aa675f1585d3ed6f1da69269a86 # v2.4 with: paths: "report.xml" show: "fail" + + - name: Slowest Tests + if: steps.changes.outputs.unit_tests == 'true' && !cancelled() + run: | + echo '## Slowest Tests' >> "$GITHUB_STEP_SUMMARY" + echo '```' >> "$GITHUB_STEP_SUMMARY" + go tool gotestsum tool slowest --num 20 --jsonfile report.json | tee -a "$GITHUB_STEP_SUMMARY" + echo '```' >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/unit_test_evalengine_mysql57.yml b/.github/workflows/unit_test_evalengine_mysql57.yml index c3677a796bb..5cd31c22893 100644 --- a/.github/workflows/unit_test_evalengine_mysql57.yml +++ b/.github/workflows/unit_test_evalengine_mysql57.yml @@ -91,9 +91,6 @@ jobs: go mod download go install golang.org/x/tools/cmd/goimports@latest - # install JUnit report formatter - go install github.com/vitessio/go-junit-report@HEAD - - name: Run make tools if: steps.changes.outputs.unit_tests == 'true' run: | @@ -127,7 +124,7 @@ jobs: # testing, e.g. MySQL 5.7 vs 8.0. export CI_DB_PLATFORM="mysql57" - make unit_test | tee -a output.txt | go-junit-report -set-exit-code > report.xml + JUNIT_OUTPUT=report.xml JSON_OUTPUT=report.json make unit_test - name: Record test results in launchable if PR is not a draft if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.changes.outputs.unit_tests == 'true' && github.base_ref == 'main' && !cancelled() @@ -135,15 +132,17 @@ jobs: # send recorded tests to launchable launchable record tests --build "$GITHUB_RUN_ID" go-test . || true - - name: Print test output - if: steps.changes.outputs.unit_tests == 'true' && !cancelled() - run: | - # print test output - cat output.txt - - name: Test Summary if: steps.changes.outputs.unit_tests == 'true' && !cancelled() uses: test-summary/action@31493c76ec9e7aa675f1585d3ed6f1da69269a86 # v2.4 with: paths: "report.xml" show: "fail" + + - name: Slowest Tests + if: steps.changes.outputs.unit_tests == 'true' && !cancelled() + run: | + echo '## Slowest Tests' >> "$GITHUB_STEP_SUMMARY" + echo '```' >> "$GITHUB_STEP_SUMMARY" + go tool gotestsum tool slowest --num 20 --jsonfile report.json | tee -a "$GITHUB_STEP_SUMMARY" + echo '```' >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/unit_test_evalengine_mysql80.yml b/.github/workflows/unit_test_evalengine_mysql80.yml index 0d989d3607c..e01048effc1 100644 --- a/.github/workflows/unit_test_evalengine_mysql80.yml +++ b/.github/workflows/unit_test_evalengine_mysql80.yml @@ -91,9 +91,6 @@ jobs: go mod download go install golang.org/x/tools/cmd/goimports@latest - # install JUnit report formatter - go install github.com/vitessio/go-junit-report@HEAD - - name: Run make tools if: steps.changes.outputs.unit_tests == 'true' run: | @@ -127,7 +124,7 @@ jobs: # testing, e.g. MySQL 5.7 vs 8.0. export CI_DB_PLATFORM="mysql80" - make unit_test | tee -a output.txt | go-junit-report -set-exit-code > report.xml + JUNIT_OUTPUT=report.xml JSON_OUTPUT=report.json make unit_test - name: Record test results in launchable if PR is not a draft if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.changes.outputs.unit_tests == 'true' && github.base_ref == 'main' && !cancelled() @@ -135,15 +132,17 @@ jobs: # send recorded tests to launchable launchable record tests --build "$GITHUB_RUN_ID" go-test . || true - - name: Print test output - if: steps.changes.outputs.unit_tests == 'true' && !cancelled() - run: | - # print test output - cat output.txt - - name: Test Summary if: steps.changes.outputs.unit_tests == 'true' && !cancelled() uses: test-summary/action@31493c76ec9e7aa675f1585d3ed6f1da69269a86 # v2.4 with: paths: "report.xml" show: "fail" + + - name: Slowest Tests + if: steps.changes.outputs.unit_tests == 'true' && !cancelled() + run: | + echo '## Slowest Tests' >> "$GITHUB_STEP_SUMMARY" + echo '```' >> "$GITHUB_STEP_SUMMARY" + go tool gotestsum tool slowest --num 20 --jsonfile report.json | tee -a "$GITHUB_STEP_SUMMARY" + echo '```' >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/unit_test_evalengine_mysql84.yml b/.github/workflows/unit_test_evalengine_mysql84.yml index 1081b13d9ea..432c4ec4a3e 100644 --- a/.github/workflows/unit_test_evalengine_mysql84.yml +++ b/.github/workflows/unit_test_evalengine_mysql84.yml @@ -91,9 +91,6 @@ jobs: go mod download go install golang.org/x/tools/cmd/goimports@latest - # install JUnit report formatter - go install github.com/vitessio/go-junit-report@HEAD - - name: Run make tools if: steps.changes.outputs.unit_tests == 'true' run: | @@ -127,7 +124,7 @@ jobs: # testing, e.g. MySQL 5.7 vs 8.0. export CI_DB_PLATFORM="mysql84" - make unit_test | tee -a output.txt | go-junit-report -set-exit-code > report.xml + JUNIT_OUTPUT=report.xml JSON_OUTPUT=report.json make unit_test - name: Record test results in launchable if PR is not a draft if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.changes.outputs.unit_tests == 'true' && github.base_ref == 'main' && !cancelled() @@ -135,15 +132,17 @@ jobs: # send recorded tests to launchable launchable record tests --build "$GITHUB_RUN_ID" go-test . || true - - name: Print test output - if: steps.changes.outputs.unit_tests == 'true' && !cancelled() - run: | - # print test output - cat output.txt - - name: Test Summary if: steps.changes.outputs.unit_tests == 'true' && !cancelled() uses: test-summary/action@31493c76ec9e7aa675f1585d3ed6f1da69269a86 # v2.4 with: paths: "report.xml" show: "fail" + + - name: Slowest Tests + if: steps.changes.outputs.unit_tests == 'true' && !cancelled() + run: | + echo '## Slowest Tests' >> "$GITHUB_STEP_SUMMARY" + echo '```' >> "$GITHUB_STEP_SUMMARY" + go tool gotestsum tool slowest --num 20 --jsonfile report.json | tee -a "$GITHUB_STEP_SUMMARY" + echo '```' >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/unit_test_mysql57.yml b/.github/workflows/unit_test_mysql57.yml index c6bb1d07dbd..510140b510f 100644 --- a/.github/workflows/unit_test_mysql57.yml +++ b/.github/workflows/unit_test_mysql57.yml @@ -91,9 +91,6 @@ jobs: go mod download go install golang.org/x/tools/cmd/goimports@latest - # install JUnit report formatter - go install github.com/vitessio/go-junit-report@HEAD - - name: Run make tools if: steps.changes.outputs.unit_tests == 'true' run: | @@ -127,7 +124,7 @@ jobs: # testing, e.g. MySQL 5.7 vs 8.0. export CI_DB_PLATFORM="mysql57" - make unit_test | tee -a output.txt | go-junit-report -set-exit-code > report.xml + JUNIT_OUTPUT=report.xml JSON_OUTPUT=report.json make unit_test - name: Record test results in launchable if PR is not a draft if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.changes.outputs.unit_tests == 'true' && github.base_ref == 'main' && !cancelled() @@ -135,15 +132,17 @@ jobs: # send recorded tests to launchable launchable record tests --build "$GITHUB_RUN_ID" go-test . || true - - name: Print test output - if: steps.changes.outputs.unit_tests == 'true' && !cancelled() - run: | - # print test output - cat output.txt - - name: Test Summary if: steps.changes.outputs.unit_tests == 'true' && !cancelled() uses: test-summary/action@31493c76ec9e7aa675f1585d3ed6f1da69269a86 # v2.4 with: paths: "report.xml" show: "fail" + + - name: Slowest Tests + if: steps.changes.outputs.unit_tests == 'true' && !cancelled() + run: | + echo '## Slowest Tests' >> "$GITHUB_STEP_SUMMARY" + echo '```' >> "$GITHUB_STEP_SUMMARY" + go tool gotestsum tool slowest --num 20 --jsonfile report.json | tee -a "$GITHUB_STEP_SUMMARY" + echo '```' >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/unit_test_mysql80.yml b/.github/workflows/unit_test_mysql80.yml index 1d0e9cceeed..5317027aa76 100644 --- a/.github/workflows/unit_test_mysql80.yml +++ b/.github/workflows/unit_test_mysql80.yml @@ -91,9 +91,6 @@ jobs: go mod download go install golang.org/x/tools/cmd/goimports@latest - # install JUnit report formatter - go install github.com/vitessio/go-junit-report@HEAD - - name: Run make tools if: steps.changes.outputs.unit_tests == 'true' run: | @@ -127,7 +124,7 @@ jobs: # testing, e.g. MySQL 5.7 vs 8.0. export CI_DB_PLATFORM="mysql80" - make unit_test | tee -a output.txt | go-junit-report -set-exit-code > report.xml + JUNIT_OUTPUT=report.xml JSON_OUTPUT=report.json make unit_test - name: Record test results in launchable if PR is not a draft if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.changes.outputs.unit_tests == 'true' && github.base_ref == 'main' && !cancelled() @@ -135,15 +132,17 @@ jobs: # send recorded tests to launchable launchable record tests --build "$GITHUB_RUN_ID" go-test . || true - - name: Print test output - if: steps.changes.outputs.unit_tests == 'true' && !cancelled() - run: | - # print test output - cat output.txt - - name: Test Summary if: steps.changes.outputs.unit_tests == 'true' && !cancelled() uses: test-summary/action@31493c76ec9e7aa675f1585d3ed6f1da69269a86 # v2.4 with: paths: "report.xml" show: "fail" + + - name: Slowest Tests + if: steps.changes.outputs.unit_tests == 'true' && !cancelled() + run: | + echo '## Slowest Tests' >> "$GITHUB_STEP_SUMMARY" + echo '```' >> "$GITHUB_STEP_SUMMARY" + go tool gotestsum tool slowest --num 20 --jsonfile report.json | tee -a "$GITHUB_STEP_SUMMARY" + echo '```' >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/unit_test_mysql84.yml b/.github/workflows/unit_test_mysql84.yml index e9e784419f0..8264c1f7c13 100644 --- a/.github/workflows/unit_test_mysql84.yml +++ b/.github/workflows/unit_test_mysql84.yml @@ -91,9 +91,6 @@ jobs: go mod download go install golang.org/x/tools/cmd/goimports@latest - # install JUnit report formatter - go install github.com/vitessio/go-junit-report@HEAD - - name: Run make tools if: steps.changes.outputs.unit_tests == 'true' run: | @@ -127,7 +124,7 @@ jobs: # testing, e.g. MySQL 5.7 vs 8.0. export CI_DB_PLATFORM="mysql84" - make unit_test | tee -a output.txt | go-junit-report -set-exit-code > report.xml + JUNIT_OUTPUT=report.xml JSON_OUTPUT=report.json make unit_test - name: Record test results in launchable if PR is not a draft if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.changes.outputs.unit_tests == 'true' && github.base_ref == 'main' && !cancelled() @@ -135,15 +132,17 @@ jobs: # send recorded tests to launchable launchable record tests --build "$GITHUB_RUN_ID" go-test . || true - - name: Print test output - if: steps.changes.outputs.unit_tests == 'true' && !cancelled() - run: | - # print test output - cat output.txt - - name: Test Summary if: steps.changes.outputs.unit_tests == 'true' && !cancelled() uses: test-summary/action@31493c76ec9e7aa675f1585d3ed6f1da69269a86 # v2.4 with: paths: "report.xml" show: "fail" + + - name: Slowest Tests + if: steps.changes.outputs.unit_tests == 'true' && !cancelled() + run: | + echo '## Slowest Tests' >> "$GITHUB_STEP_SUMMARY" + echo '```' >> "$GITHUB_STEP_SUMMARY" + go tool gotestsum tool slowest --num 20 --jsonfile report.json | tee -a "$GITHUB_STEP_SUMMARY" + echo '```' >> "$GITHUB_STEP_SUMMARY" diff --git a/test/templates/unit_test.tpl b/test/templates/unit_test.tpl index 8e7ea5a6cc8..85c9c664280 100644 --- a/test/templates/unit_test.tpl +++ b/test/templates/unit_test.tpl @@ -104,9 +104,6 @@ jobs: go mod download go install golang.org/x/tools/cmd/goimports@latest - # install JUnit report formatter - go install github.com/vitessio/go-junit-report@HEAD - - name: Run make tools if: steps.changes.outputs.unit_tests == 'true' run: | @@ -140,7 +137,7 @@ jobs: # testing, e.g. MySQL 5.7 vs 8.0. export CI_DB_PLATFORM="{{.Platform}}" - make {{if .Race}}unit_test_race{{else}}unit_test{{end}} | tee -a output.txt | go-junit-report -set-exit-code > report.xml + JUNIT_OUTPUT=report.xml JSON_OUTPUT=report.json make {{if .Race}}unit_test_race{{else}}unit_test{{end}} - name: Record test results in launchable if PR is not a draft if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.changes.outputs.unit_tests == 'true' && github.base_ref == 'main' && !cancelled() @@ -148,15 +145,17 @@ jobs: # send recorded tests to launchable launchable record tests --build "$GITHUB_RUN_ID" go-test . || true - - name: Print test output - if: steps.changes.outputs.unit_tests == 'true' && !cancelled() - run: | - # print test output - cat output.txt - - name: Test Summary if: steps.changes.outputs.unit_tests == 'true' && !cancelled() uses: test-summary/action@31493c76ec9e7aa675f1585d3ed6f1da69269a86 # v2.4 with: paths: "report.xml" show: "fail" + + - name: Slowest Tests + if: steps.changes.outputs.unit_tests == 'true' && !cancelled() + run: | + echo '## Slowest Tests' >> "$GITHUB_STEP_SUMMARY" + echo '```' >> "$GITHUB_STEP_SUMMARY" + go tool gotestsum tool slowest --num 20 --jsonfile report.json | tee -a "$GITHUB_STEP_SUMMARY" + echo '```' >> "$GITHUB_STEP_SUMMARY" diff --git a/tools/e2e_test_runner.sh b/tools/e2e_test_runner.sh index 1fc5c2cb558..8d97a89bbb4 100755 --- a/tools/e2e_test_runner.sh +++ b/tools/e2e_test_runner.sh @@ -33,7 +33,7 @@ source build.env if [[ -z $VT_GO_PARALLEL && -n $VT_GO_PARALLEL_VALUE ]]; then - VT_GO_PARALLEL="-p $VT_GO_PARALLEL_VALUE" + VT_GO_PARALLEL="-p $VT_GO_PARALLEL_VALUE" fi # All Go packages with test files. @@ -41,29 +41,29 @@ fi packages_with_tests=$(go list -f '{{if len .TestGoFiles}}{{.ImportPath}} {{join .TestGoFiles " "}}{{end}}{{if len .XTestGoFiles}}{{.ImportPath}} {{join .XTestGoFiles " "}}{{end}}' ./go/.../endtoend/... | sort) # Flaky tests have the suffix "_flaky_test.go". -all_except_flaky_and_cluster_tests=$(echo "$packages_with_tests" | grep -vE ".+ .+_flaky_test\.go" | grep -vE "go/test/endtoend" | cut -d" " -f1) +all_except_flaky_and_cluster_tests=$(echo "$packages_with_tests" | grep -vE ".+ .+_flaky_test\.go" | grep -vE "go/test/endtoend" | cut -d" " -f1) flaky_tests=$(echo "$packages_with_tests" | grep -E ".+ .+_flaky_test\.go" | grep -vE "go/test/endtoend" | cut -d" " -f1) # Run non-flaky tests. echo "$all_except_flaky_and_cluster_tests" | xargs go test -count=1 $VT_GO_PARALLEL if [ $? -ne 0 ]; then - echo "ERROR: Go unit tests failed. See above for errors." - echo - echo "This should NOT happen. Did you introduce a flaky unit test?" - echo "If so, please rename it to the suffix _flaky_test.go." - exit 1 + echo "ERROR: Go unit tests failed. See above for errors." + echo + echo "This should NOT happen. Did you introduce a flaky unit test?" + echo "If so, please rename it to the suffix _flaky_test.go." + exit 1 fi # Run flaky tests sequentially. Retry when necessary. for pkg in $flaky_tests; do - max_attempts=3 - attempt=1 - # Set a timeout because some tests may deadlock when they flake. - until go test -timeout 30s $VT_GO_PARALLEL $pkg; do - echo "FAILED (try $attempt/$max_attempts) in $pkg (return code $?). See above for errors." - if [ $((++attempt)) -gt $max_attempts ]; then - echo "ERROR: Flaky Go unit tests in package $pkg failed too often (after $max_attempts retries). Please reduce the flakiness." - exit 1 - fi - done + max_attempts=3 + attempt=1 + # Set a timeout because some tests may deadlock when they flake. + until go test -timeout 30s $VT_GO_PARALLEL $pkg; do + echo "FAILED (try $attempt/$max_attempts) in $pkg (return code $?). See above for errors." + if [ $((++attempt)) -gt $max_attempts ]; then + echo "ERROR: Flaky Go unit tests in package $pkg failed too often (after $max_attempts retries). Please reduce the flakiness." + exit 1 + fi + done done diff --git a/tools/unit_test_runner.sh b/tools/unit_test_runner.sh index 78a852e2cca..f9dadc74ddc 100755 --- a/tools/unit_test_runner.sh +++ b/tools/unit_test_runner.sh @@ -1,30 +1,21 @@ #!/bin/bash # Copyright 2019 The Vitess Authors. -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# Custom Go unit test runner which runs all unit tests in parallel except for -# known flaky unit tests. -# Flaky unit tests are run sequentially in the second phase and retried up to -# three times. - -# Why are there flaky unit tests? -# -# Some of the Go unit tests are inherently flaky e.g. because they use the -# real timer implementation and might fail when things take longer as usual. -# In particular, this happens when the system is under load and threads do not -# get scheduled as fast as usual. Then, the expected timings do not match. +# Custom Go unit test runner which runs all unit tests using gotestsum. Failed tests are +# automatically retried up to 3 times to handle flaky tests. # Set VT_GO_PARALLEL variable in the same way as the Makefile does. # We repeat this here because this script is called directly by test.go @@ -33,7 +24,13 @@ source build.env if [[ -z $VT_GO_PARALLEL && -n $VT_GO_PARALLEL_VALUE ]]; then - VT_GO_PARALLEL="-p $VT_GO_PARALLEL_VALUE" + VT_GO_PARALLEL="-p $VT_GO_PARALLEL_VALUE" +fi + +# Enable race detector if RACE=1 +RACE_FLAG="" +if [[ "$RACE" == "1" ]]; then + RACE_FLAG="-race" fi # Enable race detector if RACE=1 @@ -46,54 +43,35 @@ fi # sockets in those directories. Tell Golang to use /tmp/vttest_XXXXXX instead. kernel="$(uname -s)" case "$kernel" in - darwin|Darwin) - TMPDIR=${TMPDIR:-} - if [ -z "$TMPDIR" ]; then - TMPDIR="$(mktemp -d /tmp/vttest_XXXXXX)" - export TMPDIR - fi - echo "Using temporary directory for tests: $TMPDIR" - ;; +darwin | Darwin) + TMPDIR=${TMPDIR:-} + if [ -z "$TMPDIR" ]; then + TMPDIR="$(mktemp -d /tmp/vttest_XXXXXX)" + export TMPDIR + fi + echo "Using temporary directory for tests: $TMPDIR" + ;; esac -# All Go packages with test files. -# Output per line: * -packages_with_tests=$(go list -f '{{if len .TestGoFiles}}{{.ImportPath}} {{join .TestGoFiles " "}}{{end}}{{if len .XTestGoFiles}}{{.ImportPath}} {{join .XTestGoFiles " "}}{{end}}' ./go/... | sort) +# All Go packages with test files, excluding endtoend tests. +packages_with_tests=$(go list ./go/... | grep -v endtoend) if [[ "$VTEVALENGINETEST" == "1" ]]; then - packages_with_tests=$(echo "$packages_with_tests" | grep "evalengine") + packages_with_tests=$(echo "$packages_with_tests" | grep "evalengine") fi if [[ "$VTEVALENGINETEST" == "0" ]]; then - packages_with_tests=$(echo "$packages_with_tests" | grep -v "evalengine") + packages_with_tests=$(echo "$packages_with_tests" | grep -v "evalengine") fi -# Flaky tests have the suffix "_flaky_test.go". -# Exclude endtoend tests -all_except_flaky_tests=$(echo "$packages_with_tests" | grep -vE ".+ .+_flaky_test\.go" | cut -d" " -f1 | grep -v "endtoend") -flaky_tests=$(echo "$packages_with_tests" | grep -E ".+ .+_flaky_test\.go" | cut -d" " -f1) - -go test $VT_GO_PARALLEL $RACE_FLAG -v -count=1 $all_except_flaky_tests -if [ $? -ne 0 ]; then - echo "ERROR: Go unit tests failed. See above for errors." - echo - echo "This should NOT happen. Did you introduce a flaky unit test?" - echo "If so, please rename it to the suffix _flaky_test.go." - exit 1 +# Build gotestsum args. Failed tests are retried up to 3 times, but if more than 10 tests fail +# initially we skip retries to avoid wasting time on a real widespread failure. +GOTESTSUM_ARGS="--format github-actions --rerun-fails=3 --rerun-fails-max-failures=10 --rerun-fails-run-root-test --format-hide-empty-pkg --hide-summary=skipped" +if [[ -n "${JUNIT_OUTPUT:-}" ]]; then + GOTESTSUM_ARGS="$GOTESTSUM_ARGS --junitfile $JUNIT_OUTPUT" +fi +if [[ -n "${JSON_OUTPUT:-}" ]]; then + GOTESTSUM_ARGS="$GOTESTSUM_ARGS --jsonfile $JSON_OUTPUT" fi -echo '# Flaky tests (3 attempts permitted)' - -# Run flaky tests sequentially. Retry when necessary. -for pkg in $flaky_tests; do - max_attempts=3 - attempt=1 - # Set a timeout because some tests may deadlock when they flake. - until go test -timeout 5m $VT_GO_PARALLEL $RACE_FLAG $pkg -v -count=1; do - echo "FAILED (try $attempt/$max_attempts) in $pkg (return code $?). See above for errors." - if [ $((++attempt)) -gt $max_attempts ]; then - echo "ERROR: Flaky Go unit tests in package $pkg failed too often (after $max_attempts retries). Please reduce the flakiness." - exit 1 - fi - done -done +go tool gotestsum $GOTESTSUM_ARGS --packages="$packages_with_tests" -- $VT_GO_PARALLEL $RACE_FLAG -count=1