From 0fd1d09c9dd508860fab342e99c65d957bf94e22 Mon Sep 17 00:00:00 2001 From: Adriano Cunha <35786489+adrcunha@users.noreply.github.com> Date: Wed, 9 Jan 2019 10:54:42 -0800 Subject: [PATCH] Create default build/unit test runners. (#364) Bonuses: * lint and link check markdown files in the default build test runner * explicitly state if tests passed on failed * add support to custom pre/post build/unit tests * simplify and expand unit tests for `presubmit-tests.sh` * also build go code containing build tags in the default build test runner * update documentation missed in #362 --- scripts/README.md | 161 ++++++------ scripts/presubmit-tests.sh | 229 +++++++++++++----- test/presubmit-tests.sh | 9 +- test/unit/presubmit-custom-runners-tests.sh | 49 ---- ...presubmit-full-custom-integration-tests.sh | 28 --- .../presubmit-integration-tests-common.sh | 44 ---- ...submit-partial-custom-integration-tests.sh | 33 --- test/unit/presubmit-tests.sh | 144 +++++++++++ test/unit/release-tests.sh | 16 -- test/unit/test-helper.sh | 24 ++ 10 files changed, 431 insertions(+), 306 deletions(-) delete mode 100755 test/unit/presubmit-custom-runners-tests.sh delete mode 100755 test/unit/presubmit-full-custom-integration-tests.sh delete mode 100755 test/unit/presubmit-integration-tests-common.sh delete mode 100755 test/unit/presubmit-partial-custom-integration-tests.sh create mode 100755 test/unit/presubmit-tests.sh diff --git a/scripts/README.md b/scripts/README.md index 3c6c9aacb5f0c..dcf7dc977ee31 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -9,21 +9,35 @@ This is a helper script to run the presubmit tests. To use it: 1. Source this script. -1. Define the functions `build_tests()` and `unit_tests()`. They should run all -tests (i.e., not fail fast), and return 0 if all passed, 1 if a failure -occurred. The environment variables `RUN_BUILD_TESTS`, `RUN_UNIT_TESTS` and -`RUN_INTEGRATION_TESTS` are set to 0 (false) or 1 (true) accordingly. If -`--emit-metrics` is passed, `EMIT_METRICS` will be set to 1. - -1. [optional] Define the function `integration_tests()`, just like the previous -ones. If you don't define this function, the default action for running the -integration tests is to call the `./test/e2e-tests.sh` script (passing the -`--emit-metrics` flag if necessary). - -1. [optional] Define the functions `pre_integration_tests()` or -`post_integration_tests()`. These functions will be called before or after the -integration tests (either your custom one or the default action) and will cause -the test to fail if they don't return success. +1. [optional] Define the function `build_tests()`. If you don't define this + function, the default action for running the build tests is to: + - lint and link check markdown files + - run `go build` on the entire repo + - run `/hack/verify-codegen.sh` (if it exists) + - check licenses in `/cmd` (if it exists) + +1. [optional] Define the functions `pre_build_tests()` and/or + `post_build_tests()`. These functions will be called before or after the + build tests (either your custom one or the default action) and will cause + the test to fail if they don't return success. + +1. [optional] Define the function `unit_tests()`. If you don't define this + function, the default action for running the unit tests is to run all go tests + in the repo. + +1. [optional] Define the functions `pre_unit_tests()` and/or + `post_unit_tests()`. These functions will be called before or after the + unit tests (either your custom one or the default action) and will cause + the test to fail if they don't return success. + +1. [optional] Define the function `integration_tests()`. If you don't define + this function, the default action for running the integration tests is to run + all run all `./test/e2e-*tests.sh` scripts, in sequence. + +1. [optional] Define the functions `pre_integration_tests()` and/or + `post_integration_tests()`. These functions will be called before or after the + integration tests (either your custom one or the default action) and will cause + the test to fail if they don't return success. 1. Call the `main()` function passing `$@` (without quotes). @@ -33,20 +47,27 @@ integration tests). Use the flags `--build-tests`, `--unit-tests` and `--integration-tests` to run a specific set of tests. The flag `--emit-metrics` is used to emit metrics when -running the tests, and is automatically handled by the default action (see -above). +running the tests, and is automatically handled by the default action for +integration tests (see above). + +The script will automatically skip all presubmit tests for PRs where all changed +files are exempt of tests (e.g., a PR changing only the `OWNERS` file). + +Also, for PRs touching only markdown files, the unit and integration tests are +skipped. ### Sample presubmit test script ```bash source vendor/github.com/knative/test-infra/scripts/presubmit-tests.sh -function build_tests() { - go build . +function post_build_tests() { + echo "Cleaning up after build tests" + rm -fr ./build-cache } function unit_tests() { - report_go_test . + make -C tests test } function pre_integration_tests() { @@ -66,43 +87,44 @@ This is a helper script for Knative E2E test scripts. To use it: 1. Source the script. 1. [optional] Write the `teardown()` function, which will tear down your test -resources. + resources. 1. [optional] Write the `dump_extra_cluster_state()` function. It will be -called when a test fails, and can dump extra information about the current state -of the cluster (tipically using `kubectl`). + called when a test fails, and can dump extra information about the current state + of the cluster (tipically using `kubectl`). 1. [optional] Write the `parse_flags()` function. It will be called whenever an -unrecognized flag is passed to the script, allowing you to define your own flags. -The function must return 0 if the flag is unrecognized, or the number of items -to skip in the command line if the flag was parsed successfully. For example, -return 1 for a simple flag, and 2 for a flag with a parameter. + unrecognized flag is passed to the script, allowing you to define your own flags. + The function must return 0 if the flag is unrecognized, or the number of items + to skip in the command line if the flag was parsed successfully. For example, + return 1 for a simple flag, and 2 for a flag with a parameter. 1. Call the `initialize()` function passing `$@` (without quotes). 1. Write logic for the end-to-end tests. Run all go tests using `go_test_e2e()` -(or `report_go_test()` if you need a more fine-grained control) and call -`fail_test()` or `success()` if any of them failed. The environment variables -`DOCKER_REPO_OVERRIDE`, `K8S_CLUSTER_OVERRIDE` and `K8S_USER_OVERRIDE` will be set -according to the test cluster. You can also use the following boolean (0 is false, -1 is true) environment variables for the logic: - * `EMIT_METRICS`: true if `--emit-metrics` was passed. - * `USING_EXISTING_CLUSTER`: true if the test cluster is an already existing one, -and not a temporary cluster created by `kubetest`. + (or `report_go_test()` if you need a more fine-grained control) and call + `fail_test()` or `success()` if any of them failed. The environment variables + `DOCKER_REPO_OVERRIDE`, `K8S_CLUSTER_OVERRIDE` and `K8S_USER_OVERRIDE` will be + set according to the test cluster. You can also use the following boolean (0 is + false, 1 is true) environment variables for the logic: + + - `EMIT_METRICS`: true if `--emit-metrics` was passed. + - `USING_EXISTING_CLUSTER`: true if the test cluster is an already existing one, + and not a temporary cluster created by `kubetest`. - All environment variables above are marked read-only. + All environment variables above are marked read-only. **Notes:** 1. Calling your script without arguments will create a new cluster in the GCP -project `$PROJECT_ID` and run the tests against it. + project `$PROJECT_ID` and run the tests against it. 1. Calling your script with `--run-tests` and the variables `K8S_CLUSTER_OVERRIDE`, -`K8S_USER_OVERRIDE` and `DOCKER_REPO_OVERRIDE` set will immediately start the -tests against the cluster. + `K8S_USER_OVERRIDE` and `DOCKER_REPO_OVERRIDE` set will immediately start the + tests against the cluster. 1. You can force running the tests against a specific GKE cluster version by using -the `--cluster-version` flag and passing a X.Y.Z version as the flag value. + the `--cluster-version` flag and passing a X.Y.Z version as the flag value. ### Sample end-to-end test script @@ -150,37 +172,38 @@ This is a helper script for Knative release scripts. To use it: 1. Call the `initialize()` function passing `$@` (without quotes). 1. Call the `run_validation_tests()` function passing the script or executable that -runs the release validation tests. It will call the script to run the tests unless -`--skip_tests` was passed. + runs the release validation tests. It will call the script to run the tests unless + `--skip_tests` was passed. 1. Write logic for the release process. Call `publish_yaml()` to publish the manifest(s), -`tag_releases_in_yaml()` to tag the generated images, `branch_release()` to branch -named releases. Use the following boolean (0 is false, 1 is true) and string environment -variables for the logic: - * `RELEASE_VERSION`: contains the release version if `--version` was passed. This -also overrides the value of the `TAG` variable as `v`. - * `RELEASE_BRANCH`: contains the release branch if `--branch` was passed. Otherwise -it's empty and `master` HEAD will be considered the release branch. - * `RELEASE_NOTES`: contains the filename with the release notes if `--release-notes` -was passed. The release notes is a simple markdown file. - * `RELEASE_GCS_BUCKET`: contains the GCS bucket name to store the manifests if -`--release-gcs` was passed, otherwise the default value `knative-nightly/` will be -used. It is empty if `--publish` was not passed. - * `KO_DOCKER_REPO`: contains the GCR to store the images if `--release-gcr` was -passed, otherwise the default value `gcr.io/knative-nightly` will be used. It is set to -`ko.local` if `--publish` was not passed. - * `SKIP_TESTS`: true if `--skip-tests` was passed. This is handled automatically -by the `run_validation_tests()` function. - * `TAG_RELEASE`: true if `--tag-release` was passed. In this case, the environment -variable `TAG` will contain the release tag in the form `vYYYYMMDD-`. - * `PUBLISH_RELEASE`: true if `--publish` was passed. In this case, the environment -variable `KO_FLAGS` will be updated with the `-L` option. - * `BRANCH_RELEASE`: true if both `--version` and `--publish-release` were passed. - - All boolean environment variables default to false for safety. - - All environment variables above, except `KO_FLAGS`, are marked read-only once -`initialize()` is called. + `tag_releases_in_yaml()` to tag the generated images, `branch_release()` to branch + named releases. Use the following boolean (0 is false, 1 is true) and string environment + variables for the logic: + + - `RELEASE_VERSION`: contains the release version if `--version` was passed. This + also overrides the value of the `TAG` variable as `v`. + - `RELEASE_BRANCH`: contains the release branch if `--branch` was passed. Otherwise + it's empty and `master` HEAD will be considered the release branch. + - `RELEASE_NOTES`: contains the filename with the release notes if `--release-notes` + was passed. The release notes is a simple markdown file. + - `RELEASE_GCS_BUCKET`: contains the GCS bucket name to store the manifests if + `--release-gcs` was passed, otherwise the default value `knative-nightly/` + will be used. It is empty if `--publish` was not passed. + - `KO_DOCKER_REPO`: contains the GCR to store the images if `--release-gcr` was + passed, otherwise the default value `gcr.io/knative-nightly` will be used. It + is set to `ko.local` if `--publish` was not passed. + - `SKIP_TESTS`: true if `--skip-tests` was passed. This is handled automatically + by the `run_validation_tests()` function. + - `TAG_RELEASE`: true if `--tag-release` was passed. In this case, the environment + variable `TAG` will contain the release tag in the form `vYYYYMMDD-`. + - `PUBLISH_RELEASE`: true if `--publish` was passed. In this case, the environment + variable `KO_FLAGS` will be updated with the `-L` option. + - `BRANCH_RELEASE`: true if both `--version` and `--publish-release` were passed. + + All boolean environment variables default to false for safety. + + All environment variables above, except `KO_FLAGS`, are marked read-only once + `initialize()` is called. ### Sample release script diff --git a/scripts/presubmit-tests.sh b/scripts/presubmit-tests.sh index 332dd493a9385..07ca5fbef5cdf 100755 --- a/scripts/presubmit-tests.sh +++ b/scripts/presubmit-tests.sh @@ -20,111 +20,212 @@ source $(dirname ${BASH_SOURCE})/library.sh # Extensions or file patterns that don't require presubmit tests. -readonly NO_PRESUBMIT_FILES=(\.md \.png \.gitignore \.gitattributes ^OWNERS ^OWNERS_ALIASES ^AUTHORS) +readonly NO_PRESUBMIT_FILES=(\.png \.gitignore \.gitattributes ^OWNERS ^OWNERS_ALIASES ^AUTHORS) -# Options set by command-line flags. -RUN_BUILD_TESTS=0 -RUN_UNIT_TESTS=0 -RUN_INTEGRATION_TESTS=0 -EMIT_METRICS=0 +# Flag if this is a presubmit run or not. +[[ IS_PROW && -n "${PULL_PULL_SHA}" ]] && IS_PRESUBMIT=1 || IS_PRESUBMIT=0 +readonly IS_PRESUBMIT -# Exit presubmit tests if only documentation files were changed. -function exit_if_presubmit_not_required() { - if [[ -n "${PULL_PULL_SHA}" ]]; then - # On a presubmit job - local changes="$(/workspace/githubhelper -list-changed-files)" - if [[ -z "${changes}" ]]; then - header "NO CHANGED FILES REPORTED, ASSUMING IT'S AN ERROR AND RUNNING TESTS ANYWAY" - return - fi - local no_presubmit_pattern="${NO_PRESUBMIT_FILES[*]}" - local no_presubmit_pattern="\(${no_presubmit_pattern// /\\|}\)$" - echo -e "Changed files in commit ${PULL_PULL_SHA}:\n${changes}" - if [[ -z "$(echo "${changes}" | grep -v ${no_presubmit_pattern})" ]]; then - # Nothing changed other than files that don't require presubmit tests - header "Commit only contains changes that don't affect tests, skipping" - exit 0 - fi +# List of changed files on presubmit, LF separated. +CHANGED_FILES="" + +# Flags that this PR is exempt of presubmit tests. +IS_PRESUBMIT_EXEMPT_PR=0 + +# Flags that this PR contains only changes to documentation. +IS_DOCUMENTATION_PR=0 + +# Returns true if PR only contains the given file regexes. +# Parameters: $1 - file regexes, space separated. +function pr_only_contains() { + [[ -z "$(echo "${CHANGED_FILES}" | grep -v \(${1// /\\|}\)$))" ]] +} + +# List changed files in the current PR. +# This is implemented as a function so it can be mocked in unit tests. +function list_changed_files() { + /workspace/githubhelper -list-changed-files +} + +# Initialize flags and context for presubmit tests: +# CHANGED_FILES, IS_PRESUBMIT_EXEMPT_PR and IS_DOCUMENTATION_PR. +function initialize_environment() { + CHANGED_FILES="" + IS_PRESUBMIT_EXEMPT_PR=0 + IS_DOCUMENTATION_PR=0 + (( ! IS_PRESUBMIT )) && return + CHANGED_FILES="$(list_changed_files)" + if [[ -n "${CHANGED_FILES}" ]]; then + echo -e "Changed files in commit ${PULL_PULL_SHA}:\n${CHANGED_FILES}" + local no_presubmit_files="${NO_PRESUBMIT_FILES[*]}" + pr_only_contains "${no_presubmit_files}" && IS_PRESUBMIT_EXEMPT_PR=1 + pr_only_contains "\.md ${no_presubmit_files}" && IS_DOCUMENTATION_PR=1 + else + header "NO CHANGED FILES REPORTED, ASSUMING IT'S AN ERROR AND RUNNING TESTS ANYWAY" fi + readonly CHANGED_FILES + readonly IS_DOCUMENTATION_PR + readonly IS_PRESUBMIT_EXEMPT_PR } -# Run build tests. -# If there's no `build_tests` function, do: -# * `go build` on the entire repo -# * run `/hack/verify-codegen.sh` -# * check licenses in `/cmd` +# Display a pass/fail banner for a test group. +# Parameters: $1 - test group name (e.g., build) +# $2 - result (0=passed, 1=failed) +function results_banner() { + local result + [[ $2 -eq 0 ]] && result="PASSED" || result="FAILED" + header "$1 tests ${result}" +} + +# Run build tests. If there's no `build_tests` function, run the default +# build test runner. function run_build_tests() { (( ! RUN_BUILD_TESTS )) && return 0 header "Running build tests" - if function_exists build_tests; then - build_tests - return + local failed=0 + # Run pre-build tests, if any + if function_exists pre_build_tests; then + pre_build_tests || failed=1 + fi + # Don't run build tests if pre-build tests failed + if (( ! failed )); then + if function_exists build_tests; then + build_tests || failed=1 + else + default_build_test_runner || failed=1 + fi fi + # Don't run post-build tests if pre/build tests failed + if function_exists post_build_tests; then + post_build_tests || failed=1 + fi + results_banner "Build" ${failed} + return ${failed} +} + +# Default build test runner that: +# * lint and link check markdown files +# * `go build` on the entire repo +# * run `/hack/verify-codegen.sh` (if it exists) +# * check licenses in `/cmd` (if it exists) +function default_build_test_runner() { local failed=0 + local mdfiles="$(echo "${CHANGED_FILES}" | grep \.md$)" + if [[ -n "${mdfiles}" ]]; then + subheader "Linting the markdown files" + lint_markdown ${mdfiles} || failed=1 + subheader "Checking links in the markdown files" + check_links_in_markdown ${mdfiles} || failed=1 + fi + # For documentation PRs, just check the md files + (( IS_DOCUMENTATION_PR )) && return ${failed} + # Ensure all the code builds + subheader "Checking that go code builds" go build -v ./... || failed=1 - subheader "Checking autogenerated code is up-to-date" - ./hack/verify-codegen.sh || failed=1 + # Get all build tags in go code (ignore /vendor) + local tags="$(grep -r '// +build' . \ + | grep -v '^./vendor/' | cut -f3 -d' ' | sort | uniq | tr '\n' ' ')" + if [[ -n "${tags}" ]]; then + go test -run=^$ -tags="${tags}" ./... || failed=1 + fi + if [[ -f ./hack/verify-codegen.sh ]]; then + subheader "Checking autogenerated code is up-to-date" + ./hack/verify-codegen.sh || failed=1 + fi # Check that we don't have any forbidden licenses in our images. - subheader "Checking for forbidden licenses" - check_licenses ./cmd/* || failed=1 + if [[ -d ./cmd ]]; then + subheader "Checking for forbidden licenses" + check_licenses ./cmd/* || failed=1 + fi return ${failed} } -# Run unit tests. -# If there's no `unit_tests` function, run `go test` on the entire repo. +# Run unit tests. If there's no `unit_tests` function, run the default +# unit test runner. function run_unit_tests() { (( ! RUN_UNIT_TESTS )) && return 0 header "Running unit tests" - if function_exists unit_tests; then - unit_tests - return + local failed=0 + # Run pre-unit tests, if any + if function_exists pre_unit_tests; then + pre_unit_tests || failed=1 + fi + # Don't run unit tests if pre-unit tests failed + if (( ! failed )); then + if function_exists unit_tests; then + unit_tests || failed=1 + else + default_unit_test_runner || failed=1 + fi fi + # Don't run post-unit tests if pre/unit tests failed + if function_exists post_unit_tests; then + post_unit_tests || failed=1 + fi + results_banner "Unit" ${failed} + return ${failed} +} + +# Default unit test runner that runs all go tests in the repo. +function default_unit_test_runner() { report_go_test ./... } -# Run integration tests. -# If there's no `integration_tests` function, run all `test/e2e-*tests.sh`. +# Run integration tests. If there's no `integration_tests` function, run the +# default integration test runner. function run_integration_tests() { + # Don't run integration tests if not requested OR on documentation PRs (( ! RUN_INTEGRATION_TESTS )) && return 0 + (( IS_DOCUMENTATION_PR )) && return 0 header "Running integration tests" - local e2e_failed=0 local failed=0 # Run pre-integration tests, if any if function_exists pre_integration_tests; then - if ! pre_integration_tests; then - failed=1 - e2e_failed=1 - fi + pre_integration_tests || failed=1 fi # Don't run integration tests if pre-integration tests failed - if (( ! e2e_failed )); then + if (( ! failed )); then if function_exists integration_tests; then - if ! integration_tests; then - failed=1 - e2e_failed=1 - fi + integration_tests || failed=1 else - local options="" - (( EMIT_METRICS )) && options="--emit-metrics" - for e2e_test in ./test/e2e-*tests.sh; do - echo "Running integration test ${e2e_test}" - if ! ${e2e_test} ${options}; then - failed=1 - e2e_failed=1 - fi - done + default_integration_test_runner || failed=1 fi fi - # Don't run post-integration - if (( ! e2e_failed )) && function_exists post_integration_tests; then + # Don't run integration tests if pre/integration tests failed + if (( ! failed )) && function_exists post_integration_tests; then post_integration_tests || failed=1 fi + results_banner "Integration" ${failed} return ${failed} } +# Default integration test runner that runs all `test/e2e-*tests.sh`. +function default_integration_test_runner() { + local options="" + local failed=0 + (( EMIT_METRICS )) && options="--emit-metrics" + for e2e_test in ./test/e2e-*tests.sh; do + echo "Running integration test ${e2e_test}" + if ! ${e2e_test} ${options}; then + failed=1 + fi + done + return ${failed} +} + +# Options set by command-line flags. +RUN_BUILD_TESTS=0 +RUN_UNIT_TESTS=0 +RUN_INTEGRATION_TESTS=0 +EMIT_METRICS=0 + # Process flags and run tests accordingly. function main() { - exit_if_presubmit_not_required + initialize_environment + if (( IS_PRESUBMIT_EXEMPT_PR )) && (( ! IS_DOCUMENTATION_PR )); then + header "Commit only contains changes that don't require tests, skipping" + exit 0 + fi # Show the version of the tools we're using if (( IS_PROW )); then diff --git a/test/presubmit-tests.sh b/test/presubmit-tests.sh index e4817cb8bcf17..2d8d056874786 100755 --- a/test/presubmit-tests.sh +++ b/test/presubmit-tests.sh @@ -23,7 +23,9 @@ source $(dirname $0)/../scripts/presubmit-tests.sh -function build_tests() { +# Run our custom build tests after the standard build tests. + +function post_build_tests() { local failed=0 for dir in ci/prow ci/testgrid; do make -C ${dir} test || failed=1 @@ -35,13 +37,14 @@ function build_tests() { return ${failed} } -function unit_tests() { +# Run our custom unit tests after the standard unit tests. + +function post_unit_tests() { local failed=0 for test in ./test/unit/*-tests.sh; do subheader "Running tests in ${test}" ${test} || failed=1 done - report_go_test -count=1 ./... || failed=1 return ${failed} } diff --git a/test/unit/presubmit-custom-runners-tests.sh b/test/unit/presubmit-custom-runners-tests.sh deleted file mode 100755 index 54c6b960ab223..0000000000000 --- a/test/unit/presubmit-custom-runners-tests.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/bash - -# Copyright 2018 The Knative Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -source $(dirname $0)/presubmit-integration-tests-common.sh - -function build_tests() { - RAN_BUILD_TESTS=1 - return 0 -} - -function unit_tests() { - RAN_UNIT_TESTS=1 - return 0 -} - -function integration_tests() { - RAN_INTEGRATION_TESTS=1 - return 0 -} - -RAN_BUILD_TESTS=0 -RAN_UNIT_TESTS=0 -RAN_INTEGRATION_TESTS=0 - -trap check_results EXIT - -function check_results() { - (( RAN_BUILD_TESTS )) || test_failed "Build tests did not run" - (( RAN_UNIT_TESTS )) || test_failed "Unit tests did not run" - (( RAN_INTEGRATION_TESTS )) || test_failed "Integration tests did not run" - echo ">> All tests passed" -} - -echo ">> Testing custom test runners" - -main $@ diff --git a/test/unit/presubmit-full-custom-integration-tests.sh b/test/unit/presubmit-full-custom-integration-tests.sh deleted file mode 100755 index ebd3474625e76..0000000000000 --- a/test/unit/presubmit-full-custom-integration-tests.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash - -# Copyright 2018 The Knative Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -source $(dirname $0)/presubmit-integration-tests-common.sh - -function check_results() { - (( PRE_INTEGRATION_TESTS )) || test_failed "Pre integration tests did not run" - (( CUSTOM_INTEGRATION_TESTS )) || test_failed "Custom integration tests did not run" - (( POST_INTEGRATION_TESTS )) || test_failed "Post integration tests did not run" - echo ">> All tests passed" -} - -echo ">> Testing all custom test integration functions" - -main $@ diff --git a/test/unit/presubmit-integration-tests-common.sh b/test/unit/presubmit-integration-tests-common.sh deleted file mode 100755 index ac9ee0c11421d..0000000000000 --- a/test/unit/presubmit-integration-tests-common.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/bash - -# Copyright 2018 The Knative Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -source $(dirname $0)/test-helper.sh -source $(dirname $0)/../../scripts/presubmit-tests.sh - -function pre_integration_tests() { - PRE_INTEGRATION_TESTS=1 -} - -function integration_tests() { - CUSTOM_INTEGRATION_TESTS=1 -} - -function post_integration_tests() { - POST_INTEGRATION_TESTS=1 -} - -function build_tests() { - return 0 -} - -function unit_tests() { - return 0 -} - -PRE_INTEGRATION_TESTS=0 -CUSTOM_INTEGRATION_TESTS=0 -POST_INTEGRATION_TESTS=0 - -trap check_results EXIT diff --git a/test/unit/presubmit-partial-custom-integration-tests.sh b/test/unit/presubmit-partial-custom-integration-tests.sh deleted file mode 100755 index 75f18c3496ab0..0000000000000 --- a/test/unit/presubmit-partial-custom-integration-tests.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash - -# Copyright 2018 The Knative Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Test that pre/post integration tests don't run if unset. - -source $(dirname $0)/presubmit-integration-tests-common.sh - -function check_results() { - (( ! PRE_INTEGRATION_TESTS )) || test_failed "Pre integration tests did run" - (( CUSTOM_INTEGRATION_TESTS )) || test_failed "Custom integration tests did not run" - (( ! POST_INTEGRATION_TESTS )) || test_failed "Post integration tests did run" - echo ">> All tests passed" -} - -echo ">> Testing custom test integration function" - -unset -f pre_integration_tests -unset -f post_integration_tests - -main $@ diff --git a/test/unit/presubmit-tests.sh b/test/unit/presubmit-tests.sh new file mode 100755 index 0000000000000..336e84c08e7b4 --- /dev/null +++ b/test/unit/presubmit-tests.sh @@ -0,0 +1,144 @@ +#!/bin/bash + +# Copyright 2018 The Knative Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Fake we're in a Prow job, if running locally. +[[ -z "${PROW_JOB_ID:-}" ]] && PROW_JOB_ID=123 +[[ -z "${PULL_PULL_SHA:-}" ]] && PULL_PULL_SHA=456 +[[ -z "${ARTIFATCS:-}" ]] && ARTIFACTS=/tmp + +source $(dirname $0)/../../scripts/presubmit-tests.sh +source $(dirname $0)/test-helper.sh + +set -e + +# Mock external tools for testing purposes. + +function list_changed_files() { + echo "foobar.go" +} + +function markdown-link-check() { + return 0 +} + +function mdl() { + return 0 +} + +# Helper functions. + +function mock_presubmit_runners() { + RAN_BUILD_TESTS=0 + RAN_UNIT_TESTS=0 + RAN_INTEGRATION_TESTS=0 + PRE_BUILD_TESTS=0 + POST_BUILD_TESTS=0 + PRE_UNIT_TESTS=0 + POST_UNIT_TESTS=0 + PRE_INTEGRATION_TESTS=0 + POST_INTEGRATION_TESTS=0 + function pre_build_tests() { + PRE_BUILD_TESTS=1 + } + function build_tests() { + RAN_BUILD_TESTS=1 + } + function post_build_tests() { + POST_BUILD_TESTS=1 + } + function pre_unit_tests() { + PRE_UNIT_TESTS=1 + } + function unit_tests() { + RAN_UNIT_TESTS=1 + } + function post_unit_tests() { + POST_UNIT_TESTS=1 + } + function pre_integration_tests() { + PRE_INTEGRATION_TESTS=1 + } + function integration_tests() { + RAN_INTEGRATION_TESTS=1 + } + function post_integration_tests() { + POST_INTEGRATION_TESTS=1 + } +} + +function test_custom_runners_all() { + mock_presubmit_runners + function check_results() { + (( PRE_BUILD_TESTS )) || test_failed "Pre build tests did not run" + (( RAN_BUILD_TESTS )) || test_failed "Build tests did not run" + (( POST_BUILD_TESTS )) || test_failed "Post build tests did not run" + (( PRE_UNIT_TESTS )) || test_failed "Pre unit tests did not run" + (( RAN_UNIT_TESTS )) || test_failed "Unit tests did not run" + (( POST_UNIT_TESTS )) || test_failed "Post unit tests did not run" + (( PRE_INTEGRATION_TESTS )) || test_failed "Pre integration tests did not run" + (( RAN_INTEGRATION_TESTS )) || test_failed "Custom integration tests did not run" + (( POST_INTEGRATION_TESTS )) || test_failed "Post integration tests did not run" + echo "Test passed" + } +} + +function test_custom_runners_basic() { + mock_presubmit_runners + unset pre_build_tests + unset post_build_tests + unset pre_unit_tests + unset post_unit_tests + unset pre_integration_tests + unset post_integration_tests + function check_results() { + (( ! PRE_BUILD_TESTS )) || test_failed "Pre build tests did run" + (( RAN_BUILD_TESTS )) || test_failed "Build tests did not run" + (( ! POST_BUILD_TESTS )) || test_failed "Post build tests did run" + (( ! PRE_UNIT_TESTS )) || test_failed "Pre unit tests did run" + (( RAN_UNIT_TESTS )) || test_failed "Unit tests did not run" + (( ! POST_UNIT_TESTS )) || test_failed "Post unit tests did run" + (( ! PRE_INTEGRATION_TESTS )) || test_failed "Pre integration tests did run" + (( RAN_INTEGRATION_TESTS )) || test_failed "Custom integration tests did not run" + (( ! POST_INTEGRATION_TESTS )) || test_failed "Post integration tests did run" + echo "Test passed" + } +} + +function run_markdown_build_tests() { + function list_changed_files() { + echo "README.md" + } + main --build-tests +} + +function run_main() { + # Keep current EXIT trap, used by `test_function` + local current_trap="$(trap -p EXIT | cut -d\' -f2)" + trap -- "${current_trap};check_results" EXIT + main +} + +echo ">> Testing custom test runners" + +test_function ${SUCCESS} "Test passed" call_function_pre test_custom_runners_all run_main +test_function ${SUCCESS} "Test passed" call_function_pre test_custom_runners_basic run_main + +echo ">> Testing default test runners" + +test_function ${SUCCESS} "BUILD TESTS PASSED" main --build-tests +test_function ${SUCCESS} "BUILD TESTS PASSED" call_function_pre run_markdown_build_tests + +echo ">> All tests passed" diff --git a/test/unit/release-tests.sh b/test/unit/release-tests.sh index 8566f95b982e3..168340b607b80 100755 --- a/test/unit/release-tests.sh +++ b/test/unit/release-tests.sh @@ -32,22 +32,6 @@ function mock_branch_release() { branch_release "$@" 2>&1 } -function call_function_pre() { - set -e - local init="$1" - shift - eval ${init} - "$@" 2>&1 -} - -function call_function_post() { - set -e - local post="$1" - shift - "$@" 2>&1 - eval ${post} -} - echo ">> Testing helper functions" test_function ${SUCCESS} "0.2" master_version "v0.2.1" diff --git a/test/unit/test-helper.sh b/test/unit/test-helper.sh index ac742882ccffa..2f248ad18b12f 100644 --- a/test/unit/test-helper.sh +++ b/test/unit/test-helper.sh @@ -55,6 +55,30 @@ function test_function() { echo "'$@' returns code ${expected_retcode} and displays '${expected_string}'" } +# Test helper that calls two functions in sequence. +# Parameters: $1 - function to call first. +# $2 - function to call second. +# $3..$n - parameters passed to the second function. +function call_function_pre() { + set -e + local init="$1" + shift + eval ${init} + "$@" 2>&1 +} + +# Test helper that calls two functions in sequence. +# Parameters: $1 - function to call second. +# $2 - function to call first. +# $3..$n - parameters passed to the first function. +function call_function_post() { + set -e + local post="$1" + shift + "$@" 2>&1 + eval ${post} +} + # Run the function with gcloud mocked (does nothing and outputs nothing). # Parameters: $1..$n - parameters passed to the function. function mock_gcloud_function() {