Skip to content

Use e2e-tests.sh helper from prow-tests image #1638

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
google-prow-robot merged 1 commit into from
Jul 20, 2018
Merged

Use e2e-tests.sh helper from prow-tests image #1638

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
231 changes: 29 additions & 202 deletions test/e2e-tests.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,39 +15,31 @@
# limitations under the License.

# This script runs the end-to-end tests against Knative Serving built from source.
# It is started by prow for each PR.
# For convenience, it can also be executed manually.
# It is started by prow for each PR. For convenience, it can also be executed manually.

# If you already have the *_OVERRIDE environment variables set, call
# this script with the --run-tests arguments and it will start knative in
# the cluster and run the tests.

# Calling this script without arguments will create a new cluster in
# project $PROJECT_ID, start knative in it, run the tests and delete the
# cluster. $DOCKER_REPO_OVERRIDE must point to a valid writable docker repo.
# cluster.

source "$(dirname $(readlink -f ${BASH_SOURCE}))/library.sh"
# Load github.com/knative/test-infra/images/prow-tests/scripts/e2e-tests.sh
[ -f /workspace/e2e-tests.sh ] \
&& source /workspace/e2e-tests.sh \
|| eval "$(docker run --entrypoint sh gcr.io/knative-tests/test-infra/prow-tests -c 'cat e2e-tests.sh')"
[ -v KNATIVE_TEST_INFRA ] || exit 1

# Test cluster parameters and location of generated test images
readonly E2E_CLUSTER_NAME=knative-e2e-cluster${BUILD_NUMBER}
readonly E2E_NETWORK_NAME=knative-e2e-net${BUILD_NUMBER}
readonly E2E_CLUSTER_ZONE=us-central1-a
readonly E2E_CLUSTER_NODES=3
readonly E2E_CLUSTER_MACHINE=n1-standard-4
readonly TEST_RESULT_FILE=/tmp/knative-e2e-result
readonly ISTIO_VERSION=0.8.0
readonly ISTIO_DIR=./third_party/istio-${ISTIO_VERSION}/

# This script.
readonly SCRIPT_CANONICAL_PATH="$(readlink -f ${BASH_SOURCE})"
# Location of istio for the test cluster
readonly ISTIO_YAML=./third_party/istio-0.8.0/istio.yaml

# Helper functions.

function create_istio() {
kubectl apply -f ${ISTIO_DIR}/istio.yaml
kubectl apply -f ${ISTIO_YAML}
}


function create_monitoring() {
kubectl apply -R -f config/monitoring/100-common \
-f config/monitoring/150-elasticsearch-prod \
Expand All @@ -65,7 +57,7 @@ function create_everything() {
}

function delete_istio() {
kubectl delete -f ${ISTIO_DIR}/istio.yaml
kubectl delete -f ${ISTIO_YAML}
kubectl delete clusterrolebinding cluster-admin-binding
}

Expand All @@ -85,52 +77,20 @@ function delete_everything() {
}

function teardown() {
header "Tearing down test environment"
# Free resources in GCP project.
if (( ! USING_EXISTING_CLUSTER )); then
delete_everything
fi

# Delete Knative Serving images when using prow.
if (( IS_PROW )); then
echo "Images in ${DOCKER_REPO_OVERRIDE}:"
gcloud container images list --repository=${DOCKER_REPO_OVERRIDE}
delete_gcr_images ${DOCKER_REPO_OVERRIDE}
else
# Delete the kubernetes source downloaded by kubetest
rm -fr kubernetes kubernetes.tar.gz
fi
}

function abort_if_failed() {
[[ $? -eq 0 ]] && return 0
dump_stack_info
exit 1
delete_everything
}

function dump_stack_info() {
echo "***************************************"
echo "*** TEST FAILED ***"
echo "*** Start of information dump ***"
echo "***************************************"
echo ">>> All resources:"
kubectl get all --all-namespaces
echo ">>> Services:"
kubectl get services --all-namespaces
echo ">>> Events:"
kubectl get events --all-namespaces
function dump_extra_cluster_state() {
echo ">>> Routes:"
kubectl get routes -o yaml --all-namespaces
echo ">>> Configurations:"
kubectl get configurations -o yaml --all-namespaces
echo ">>> Revisions:"
kubectl get revisions -o yaml --all-namespaces
echo ">>> Knative Serving controller log:"
kubectl logs $(get_knative_pod controller) -n knative-serving
echo "***************************************"
echo "*** TEST FAILED ***"
echo "*** End of information dump ***"
echo "***************************************"
local controller=$(kubectl get pods -n knative-serving \
--selector=app=controller --output=jsonpath="{.items[0].metadata.name}")
kubectl logs $(get_app_pod controller knative-serving)
}

function run_e2e_tests() {
Expand All @@ -139,170 +99,37 @@ function run_e2e_tests() {
kubectl label namespace $2 istio-injection=enabled --overwrite
local options=""
(( EMIT_METRICS )) && options="-emitmetrics"
report_go_test -v -tags=e2e -count=1 ./test/$1 -dockerrepo gcr.io/knative-tests/test-images/$3 ${options}
report_go_test -v -tags=e2e -count=1 ./test/$1 -dockerrepo gcr.io/knative-tests/test-images/$1 ${options}

local result=$?
[[ ${result} -ne 0 ]] && dump_stack_info
[[ ${result} -ne 0 ]] && dump_cluster_state
return ${result}
}

# Script entry point.

cd ${SERVING_ROOT_DIR}

RUN_TESTS=0
EMIT_METRICS=0
for parameter in $@; do
case $parameter in
--run-tests)
RUN_TESTS=1
shift
;;
--emit-metrics)
EMIT_METRICS=1
shift
;;
*)
echo "error: unknown option ${parameter}"
echo "usage: $0 [--run-tests][--emit-metrics]"
exit 1
;;
esac
done
readonly RUN_TESTS
readonly EMIT_METRICS

# Create the test cluster if not running tests.

if (( ! RUN_TESTS )); then
header "Creating test cluster"
# Smallest cluster required to run the end-to-end-tests
CLUSTER_CREATION_ARGS=(
--gke-create-args="--enable-autoscaling --min-nodes=1 --max-nodes=${E2E_CLUSTER_NODES} --scopes=cloud-platform"
--gke-shape={\"default\":{\"Nodes\":${E2E_CLUSTER_NODES}\,\"MachineType\":\"${E2E_CLUSTER_MACHINE}\"}}
--provider=gke
--deployment=gke
--cluster="${E2E_CLUSTER_NAME}"
--gcp-zone="${E2E_CLUSTER_ZONE}"
--gcp-network="${E2E_NETWORK_NAME}"
--gke-environment=prod
)
if (( ! IS_PROW )); then
CLUSTER_CREATION_ARGS+=(--gcp-project=${PROJECT_ID:?"PROJECT_ID must be set to the GCP project where the tests are run."})
fi
# SSH keys are not used, but kubetest checks for their existence.
# Touch them so if they don't exist, empty files are create to satisfy the check.
touch $HOME/.ssh/google_compute_engine.pub
touch $HOME/.ssh/google_compute_engine
# Clear user and cluster variables, so they'll be set to the test cluster.
# DOCKER_REPO_OVERRIDE is not touched because when running locally it must
# be a writeable docker repo.
export K8S_USER_OVERRIDE=
export K8S_CLUSTER_OVERRIDE=
# Assume test failed (see more details at the end of this script).
echo -n "1"> ${TEST_RESULT_FILE}
test_cmd_args="--run-tests"
(( EMIT_METRICS )) && test_cmd_args+=" --emit-metrics"
kubetest "${CLUSTER_CREATION_ARGS[@]}" \
--up \
--down \
--extract "gke-${SERVING_GKE_VERSION}" \
--gcp-node-image ${SERVING_GKE_IMAGE} \
--test-cmd "${SCRIPT_CANONICAL_PATH}" \
--test-cmd-args "${test_cmd_args}"
# Delete target pools and health checks that might have leaked.
# See https://github.com/knative/serving/issues/959 for details.
# TODO(adrcunha): Remove once the leak issue is resolved.
gcp_project=${PROJECT_ID}
[[ -z ${gcp_project} ]] && gcp_project=$(gcloud config get-value project)
http_health_checks="$(gcloud compute target-pools list \
--project=${gcp_project} --format='value(healthChecks)' --filter="instances~-${E2E_CLUSTER_NAME}-" | \
grep httpHealthChecks | tr '\n' ' ')"
target_pools="$(gcloud compute target-pools list \
--project=${gcp_project} --format='value(name)' --filter="instances~-${E2E_CLUSTER_NAME}-" | \
tr '\n' ' ')"
region="$(gcloud compute zones list --filter=name=${E2E_CLUSTER_ZONE} --format='value(region)')"
if [[ -n "${target_pools}" ]]; then
echo "Found leaked target pools, deleting"
gcloud compute forwarding-rules delete -q --project=${gcp_project} --region=${region} ${target_pools}
gcloud compute target-pools delete -q --project=${gcp_project} --region=${region} ${target_pools}
fi
if [[ -n "${http_health_checks}" ]]; then
echo "Found leaked health checks, deleting"
gcloud compute http-health-checks delete -q --project=${gcp_project} ${http_health_checks}
fi
result="$(cat ${TEST_RESULT_FILE})"
echo "Test result code is $result"
exit $result
fi

# --run-tests passed, run the tests.
initialize $@

# Fail fast during setup.
set -o errexit
set -o pipefail

# Set the required variables if necessary.

if [[ -z ${K8S_USER_OVERRIDE} ]]; then
export K8S_USER_OVERRIDE=$(gcloud config get-value core/account)
fi

USING_EXISTING_CLUSTER=1
if [[ -z ${K8S_CLUSTER_OVERRIDE} ]]; then
USING_EXISTING_CLUSTER=0
export K8S_CLUSTER_OVERRIDE=$(kubectl config current-context)
acquire_cluster_admin_role ${K8S_USER_OVERRIDE} ${E2E_CLUSTER_NAME} ${E2E_CLUSTER_ZONE}
# Make sure we're in the default namespace. Currently kubetest switches to
# test-pods namespace when creating the cluster.
kubectl config set-context $K8S_CLUSTER_OVERRIDE --namespace=default
fi
readonly USING_EXISTING_CLUSTER

if [[ -z ${DOCKER_REPO_OVERRIDE} ]]; then
export DOCKER_REPO_OVERRIDE=gcr.io/$(gcloud config get-value project)/knative-e2e-img
fi
export KO_DOCKER_REPO=${DOCKER_REPO_OVERRIDE}

# Build and start Knative Serving.

echo "- Cluster is ${K8S_CLUSTER_OVERRIDE}"
echo "- User is ${K8S_USER_OVERRIDE}"
echo "- Docker is ${DOCKER_REPO_OVERRIDE}"

header "Building and starting Knative Serving"
trap teardown EXIT

if (( USING_EXISTING_CLUSTER )); then
echo "Deleting any previous Knative Serving instance"
delete_everything
fi

export KO_DOCKER_REPO=${DOCKER_REPO_OVERRIDE}
create_everything

# Handle failures ourselves, so we can dump useful info.
# Handle test failures ourselves, so we can dump useful info.
set +o errexit
set +o pipefail

wait_until_pods_running knative-serving || abort_if_failed
wait_until_pods_running istio-system || abort_if_failed
wait_until_service_has_external_ip istio-system knative-ingressgateway
abort_if_failed
wait_until_pods_running knative-serving || fail_test
wait_until_pods_running istio-system || fail_test
wait_until_service_has_external_ip istio-system knative-ingressgateway || fail_test

# Run the tests

run_e2e_tests conformance pizzaplanet conformance
result=$?
run_e2e_tests e2e noodleburg e2e
[[ $? -ne 0 || ${result} -ne 0 ]] && exit 1

# kubetest teardown might fail and thus incorrectly report failure of the
# script, even if the tests pass.
# We store the real test result to return it later, ignoring any teardown
# failure in kubetest.
# TODO(adrcunha): Get rid of this workaround.
echo -n "0"> ${TEST_RESULT_FILE}
echo "**************************************"
echo "*** ALL TESTS PASSED ***"
echo "**************************************"
exit 0
result=0
run_e2e_tests conformance pizzaplanet || result=1
run_e2e_tests e2e noodleburg || result=1
[[ ${result} -ne 0 ]] && exit 1 # run_e2e_tests already dumps state
success