diff --git a/.circleci/config.yml b/.circleci/config.yml index c0d1c0181d..ce4d45299b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -62,6 +62,23 @@ workflows: - /rel\/.*/ - /hotfix\/.*/ context: slack-secrets + - amd64_e2e_expect: + requires: + - amd64_build + filters: + branches: + ignore: + - /rel\/.*/ + - /hotfix\/.*/ + - amd64_e2e_expect_nightly: + requires: + - amd64_build + filters: + branches: + only: + - /rel\/.*/ + - /hotfix\/.*/ + context: slack-secrets - arm64_build - arm64_test: requires: @@ -114,6 +131,23 @@ workflows: - /rel\/.*/ - /hotfix\/.*/ context: slack-secrets + - arm64_e2e_expect: + requires: + - arm64_build + filters: + branches: + ignore: + - /rel\/.*/ + - /hotfix\/.*/ + - arm64_e2e_expect_nightly: + requires: + - arm64_build + filters: + branches: + only: + - /rel\/.*/ + - /hotfix\/.*/ + context: slack-secrets - mac_amd64_build - mac_amd64_test: requires: @@ -166,6 +200,23 @@ workflows: - /rel\/.*/ - /hotfix\/.*/ context: slack-secrets + - mac_amd64_e2e_expect: + requires: + - mac_amd64_build + filters: + branches: + ignore: + - /rel\/.*/ + - /hotfix\/.*/ + - mac_amd64_e2e_expect_nightly: + requires: + - mac_amd64_build + filters: + branches: + only: + - /rel\/.*/ + - /hotfix\/.*/ + context: slack-secrets #- windows_x64_build commands: @@ -473,6 +524,33 @@ jobs: event: fail template: basic_fail_1 + amd64_e2e_expect: + machine: + image: ubuntu-2004:202104-01 + resource_class: large + environment: + E2E_TEST_FILTER: "EXPECT" + steps: + - prepare_go + - generic_integration: + result_subdir: amd64-e2e_subs + short_test_flag: "-short" + + amd64_e2e_expect_nightly: + machine: + image: ubuntu-2004:202104-01 + resource_class: large + environment: + E2E_TEST_FILTER: "EXPECT" + steps: + - prepare_go + - generic_integration: + result_subdir: amd64-e2e_subs_nightly + no_output_timeout: 45m + - slack/notify: + event: fail + template: basic_fail_1 + arm64_build: machine: image: ubuntu-2004:202101-01 @@ -571,6 +649,35 @@ jobs: event: fail template: basic_fail_1 + arm64_e2e_expect: + machine: + image: ubuntu-2004:202101-01 + resource_class: arm.large + environment: + E2E_TEST_FILTER: "EXPECT" + steps: + - checkout + - prepare_go + - generic_integration: + result_subdir: arm64-e2e_subs + short_test_flag: "-short" + + arm64_e2e_expect_nightly: + machine: + image: ubuntu-2004:202101-01 + resource_class: arm.large + environment: + E2E_TEST_FILTER: "EXPECT" + steps: + - checkout + - prepare_go + - generic_integration: + result_subdir: arm64-e2e_subs-nightly + no_output_timeout: 45m + - slack/notify: + event: fail + template: basic_fail_1 + mac_amd64_build: macos: xcode: 12.0.1 @@ -687,6 +794,39 @@ jobs: event: fail template: basic_fail_1 + mac_amd64_e2e_expect: + macos: + xcode: 12.0.1 + resource_class: large + environment: + E2E_TEST_FILTER: "EXPECT" + HOMEBREW_NO_AUTO_UPDATE: "true" + steps: + #- run: git -C /usr/local/Homebrew/Library/Taps/homebrew/homebrew-core fetch --unshallow + - prepare_go + - generic_integration: + result_subdir: mac-amd64-e2e_subs + circleci_home: /Users/distiller + short_test_flag: "-short" + + mac_amd64_e2e_expect_nightly: + macos: + xcode: 12.0.1 + resource_class: large + environment: + E2E_TEST_FILTER: "EXPECT" + HOMEBREW_NO_AUTO_UPDATE: "true" + steps: + #- run: git -C /usr/local/Homebrew/Library/Taps/homebrew/homebrew-core fetch --unshallow + - prepare_go + - generic_integration: + result_subdir: mac-amd64-e2e_subs-nightly + circleci_home: /Users/distiller + no_output_timeout: 45m + - slack/notify: + event: fail + template: basic_fail_1 + windows_x64_build: executor: name: win/default diff --git a/scripts/travis/run_tests.sh b/scripts/travis/run_tests.sh index cf47b0b977..10b145575c 100755 --- a/scripts/travis/run_tests.sh +++ b/scripts/travis/run_tests.sh @@ -3,6 +3,11 @@ set -e if [ "${BUILD_TYPE}" = "integration" ]; then + # Travis has trouble with the expect tests. + if [ "${TRAVIS}" = "true" ]; then + export RUN_EXPECT="FALSE" + fi + # Run short tests when doing pull requests; leave the long testing for nightly runs. if [[ "${TRAVIS_BRANCH}" =~ ^rel/nightly ]] || [[ "${TRAVIS_BRANCH}" =~ ^hotfix/ ]]; then SHORTTEST= @@ -11,7 +16,7 @@ if [ "${BUILD_TYPE}" = "integration" ]; then fi export SHORTTEST make integration -elif [ "${TRAVIS_EVENT_TYPE}" = "cron" ] || [[ "${TRAVIS_BRANCH}" =~ ^rel/ ]] || [[ "${TRAVIS_BRANCH}" =~ ^hotfix/ ]]; then +elif [ "${TRAVIS_EVENT_TYPE}" = "cron" ] || [[ "${TRAVIS_BRANCH}" =~ ^rel/ ]] || [[ "${TRAVIS_BRANCH}" =~ ^hotfix/ ]]; then make fulltest -j2 else make shorttest -j2 diff --git a/test/e2e-go/cli/goal/expect/goal_expect_test.go b/test/e2e-go/cli/goal/expect/goal_expect_test.go index c094c4543a..ce22987f07 100644 --- a/test/e2e-go/cli/goal/expect/goal_expect_test.go +++ b/test/e2e-go/cli/goal/expect/goal_expect_test.go @@ -24,7 +24,6 @@ import ( // TestGoalWithExpect Process all expect script files with suffix Test.exp within the test/e2e-go/cli/goal/expect directory func TestGoalWithExpect(t *testing.T) { - t.Skip("goal expect test are disabled due to flakiness") et := fixtures.MakeExpectTest(t) et.Run() } diff --git a/test/framework/fixtures/expectFixture.go b/test/framework/fixtures/expectFixture.go index 2f8cdf420b..377bce5027 100644 --- a/test/framework/fixtures/expectFixture.go +++ b/test/framework/fixtures/expectFixture.go @@ -98,6 +98,10 @@ func (ef *ExpectFixture) removeTestDir(workingDir string) (err error) { // MakeExpectTest creates an expect test fixture for the current directory func MakeExpectTest(t *testing.T) *ExpectFixture { + if skipExpectTests() { + t.Skip("Expect tests disabled by environment variables.") + } + ef := &ExpectFixture{} ef.expectFiles = make(map[string]string) // map expect test to full file name. err := filepath.Walk(".", func(path string, info os.FileInfo, err error) error { @@ -112,6 +116,33 @@ func MakeExpectTest(t *testing.T) *ExpectFixture { return ef } +func skipExpectTests() bool { + // Explicitly enabled. + if strings.ToUpper(os.Getenv("RUN_EXPECT")) == "TRUE" { + return false + } + if strings.ToUpper(os.Getenv("RUN_EXPECT")) == "FALSE" { + return true + } + + // If any of the CI systems didn't set RUN_EXPECT, disable them. + if strings.ToUpper(os.Getenv("CI")) == "TRUE" { + return true + } + if strings.ToUpper(os.Getenv("CIRCLECI")) == "TRUE" { + return true + } + if strings.ToUpper(os.Getenv("TRAVIS")) == "TRUE" { + return true + } + if strings.ToUpper(os.Getenv("JENKINS_URL")) != "" { + return true + } + + // Implicitly enable for devs running tests. + return false +} + // Run Process all expect script files with suffix Test.exp within the current directory func (ef *ExpectFixture) Run() { disabledTest := map[string]string{ diff --git a/test/scripts/e2e.sh b/test/scripts/e2e.sh index 56c57a067b..594efb0c2c 100755 --- a/test/scripts/e2e.sh +++ b/test/scripts/e2e.sh @@ -45,6 +45,18 @@ TEST_RUN_ID=$(${SCRIPT_PATH}/testrunid.py) export TEMPDIR=${SRCROOT}/tmp/out/e2e/${TEST_RUN_ID} echo "Test output can be found in ${TEMPDIR}" +function cleanup() { + echo "Cleaning up temp dir." + + rm -rf "${TEMPDIR}" + + if ! ${NO_BUILD} ; then + rm -rf ${PKG_ROOT} + fi +} + +# Cleanup files created during tests. +trap cleanup EXIT # ARM64 has an unoptimized scrypt() which can cause tests to timeout. # Run kmd with scrypt() configured to run less secure and fast to go through the motions for test. @@ -101,7 +113,6 @@ export GOPATH=$(go env GOPATH) cd "${SCRIPT_PATH}" if [ -z "$E2E_TEST_FILTER" ] || [ "$E2E_TEST_FILTER" == "SCRIPTS" ]; then - ./timeout 200 ./e2e_basic_start_stop.sh duration "e2e_basic_start_stop.sh" @@ -122,28 +133,35 @@ if [ -z "$E2E_TEST_FILTER" ] || [ "$E2E_TEST_FILTER" == "SCRIPTS" ]; then for script in "$SRCROOT"/test/scripts/e2e_subs/serial/*; do "${TEMPDIR}/ve/bin/python3" e2e_client_runner.py ${RUN_KMD_WITH_UNSAFE_SCRYPT} $script done - duration "serial client runners" deactivate + duration "serial client runners" fi # if E2E_TEST_FILTER == "" or == "SCRIPTS" if [ -z "$E2E_TEST_FILTER" ] || [ "$E2E_TEST_FILTER" == "GO" ]; then # Export our root temp folder as 'TESTDIR' for tests to use as their root test folder # This allows us to clean up everything with our rm -rf trap. - export TESTDIR=${TEMPDIR} + mkdir "${TEMPDIR}/go" + export TESTDIR=${TEMPDIR}/go export TESTDATADIR=${SRCROOT}/test/testdata export SRCROOT=${SRCROOT} ./e2e_go_tests.sh ${GO_TEST_ARGS} - duration "e2e_go_tests.sh" + duration "go integration tests" +fi # if E2E_TEST_FILTER == "" or == "GO" - rm -rf "${TEMPDIR}" +if [ -z "$E2E_TEST_FILTER" ] || [ "$E2E_TEST_FILTER" == "EXPECT" ]; then + # Export our root temp folder as 'TESTDIR' for tests to use as their root test folder + # This allows us to clean up everything with our rm -rf trap. + mkdir "${TEMPDIR}/expect" + export TESTDIR=${TEMPDIR}/expect + export TESTDATADIR=${SRCROOT}/test/testdata + export SRCROOT=${SRCROOT} - if ! ${NO_BUILD} ; then - rm -rf ${PKG_ROOT} - fi + ./e2e_go_tests.sh -e ${GO_TEST_ARGS} + duration "expect tests" +fi # if E2E_TEST_FILTER == "" or == "EXPECT" - echo "----------------------------------------------------------------------" - echo " DONE: E2E" - echo "----------------------------------------------------------------------" -fi # if E2E_TEST_FILTER == "" or == "GO" +echo "----------------------------------------------------------------------" +echo " DONE: E2E" +echo "----------------------------------------------------------------------" diff --git a/test/scripts/e2e_go_tests.sh b/test/scripts/e2e_go_tests.sh index 09371bb889..56d1eb38e8 100755 --- a/test/scripts/e2e_go_tests.sh +++ b/test/scripts/e2e_go_tests.sh @@ -13,8 +13,13 @@ GOTESTCOMMAND=${GOTESTCOMMAND:="go test"} TESTPATTERNS=() NORACEBUILD="" +export RUN_EXPECT="FALSE" while [ "$1" != "" ]; do case "$1" in + -e) + # The test code checks this variable. + export RUN_EXPECT="TRUE" + ;; -t) shift TESTPATTERNS+=($1) @@ -30,6 +35,11 @@ while [ "$1" != "" ]; do shift done +if [[ -n $TESTPATTERNS && -n $RUN_EXPECT ]]; then + echo "-t and -e are mutually exclusive." + exit 1 +fi + # Anchor our repo root reference location REPO_ROOT="$( cd "$(dirname "$0")" ; pwd -P )"/../.. @@ -89,18 +99,24 @@ if [[ "${ARCHTYPE}" = arm* ]]; then PARALLEL_FLAG="-p 1" fi +PACKAGES="./..." +if [ "$RUN_EXPECT" != "" ]; then + PACKAGES=$(go list ./...|grep expect) +fi + echo "PARALLEL_FLAG = ${PARALLEL_FLAG}" +echo "PACKAGES = ${PACKAGES}" if [ "${#TESTPATTERNS[@]}" -eq 0 ]; then - ${GOTESTCOMMAND} ${RACE_OPTION} ${PARALLEL_FLAG} -timeout 1h -v ${SHORTTEST} ./... + ${GOTESTCOMMAND} ${RACE_OPTION} ${PARALLEL_FLAG} -timeout 1h -v ${SHORTTEST} ${PACKAGES} else for TEST in ${TESTPATTERNS[@]}; do - ${GOTESTCOMMAND} ${RACE_OPTION} ${PARALLEL_FLAG} -timeout 1h -v ${SHORTTEST} -run ${TEST} ./... + ${GOTESTCOMMAND} ${RACE_OPTION} ${PARALLEL_FLAG} -timeout 1h -v ${SHORTTEST} -run ${TEST} ${PACKAGES} done fi if [ ${CLEANUP_TEMPDIR} -ne 0 ]; then - rm -rf ${TEMPDIR} + rm -rf "${TEMPDIR}" fi echo "----------------------------------------------------------------------"