diff --git a/Makefile b/Makefile index 28ba398972..986013600f 100644 --- a/Makefile +++ b/Makefile @@ -20,6 +20,7 @@ elasticsearch-ephemeral minio-external cassandra-external \ nginx-ingress-controller nginx-ingress-services reaper sftd restund coturn \ inbucket k8ssandra-test-cluster KIND_CLUSTER_NAME := wire-server +HELM_PARALLELISM ?= 1 # 1 for sequential tests; 6 for all-parallel tests package ?= all EXE_SCHEMA := ./dist/$(package)-schema @@ -315,15 +316,15 @@ kube-integration: kube-integration-setup kube-integration-test .PHONY: kube-integration-setup kube-integration-setup: charts-integration - export NAMESPACE=$(NAMESPACE); ./hack/bin/integration-setup-federation.sh + export NAMESPACE=$(NAMESPACE); export HELM_PARALLELISM=$(HELM_PARALLELISM); ./hack/bin/integration-setup-federation.sh .PHONY: kube-integration-test kube-integration-test: - export NAMESPACE=$(NAMESPACE); ./hack/bin/integration-test.sh + export NAMESPACE=$(NAMESPACE); export HELM_PARALLELISM=$(HELM_PARALLELISM); ./hack/bin/integration-test.sh .PHONY: kube-integration-teardown kube-integration-teardown: - export NAMESPACE=$(NAMESPACE); ./hack/bin/integration-teardown-federation.sh + export NAMESPACE=$(NAMESPACE); export HELM_PARALLELISM=$(HELM_PARALLELISM); ./hack/bin/integration-teardown-federation.sh .PHONY: kube-integration-e2e-telepresence kube-integration-e2e-telepresence: diff --git a/changelog.d/5-internal/parallel-helm-tests b/changelog.d/5-internal/parallel-helm-tests new file mode 100644 index 0000000000..3f4909c3b7 --- /dev/null +++ b/changelog.d/5-internal/parallel-helm-tests @@ -0,0 +1 @@ +Add config to allow to run helm tests for different services in parallel; improve integration test output logs diff --git a/hack/bin/integration-logs-relevant-bits.sh b/hack/bin/integration-logs-relevant-bits.sh new file mode 100755 index 0000000000..96b836ca90 --- /dev/null +++ b/hack/bin/integration-logs-relevant-bits.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +set -euo pipefail + +USAGE=" +Filter out garbage from logs (but keep colors and highlight problems). + +(Adapted from logfilter.sh by Stefan Matting) + +To use it pipe log output from integration-test.sh into this tool. + +Usage: logs | $0 [options] +" + +red_color="\x1b\[;1m\x1b\[31m" +problem_markers="Failures:|FAILED|ExitFailure""\ +|\^+""\ +|FAIL""\ +|tests failed""\ +|Test suite failure""\ +|""$red_color""error:""\ +|""$red_color""\ +|Test suite .+ failed" + +exit_usage() { + echo "$USAGE" + exit 1 +} + +# remove debug/info logs +# often this is just noise like connection to cassandra. +excludeLogEntries() { + grep -v '^{".*Info"' | + grep -v '^{".*Debug"' | + grep -v '^20.*, D, .*socket: [0-9]\+>$' +} + +cleanup() { + # replace backspaces with newlines + # remove "Progress" lines + # Remove blank lines + # add newline between interleaved test name and log output lines + sed 's/\x08\+/\n/g' | + sed '/^Progress [0-9]\+/d' | + sed '/^\s\+$/d' | + sed 's/:\s\+{/:\n{/g' +} + +grepper() { + # print 10 lines before/after for context + rg "$problem_markers" --color=always -A 10 -B 10 + echo -e "\033[0m" +} + +cleanup | excludeLogEntries | grepper diff --git a/hack/bin/integration-setup-federation.sh b/hack/bin/integration-setup-federation.sh index 7795226bab..f569928239 100755 --- a/hack/bin/integration-setup-federation.sh +++ b/hack/bin/integration-setup-federation.sh @@ -7,6 +7,7 @@ TOP_LEVEL="$DIR/../.." export NAMESPACE=${NAMESPACE:-test-integration} HELMFILE_ENV=${HELMFILE_ENV:-default} CHARTS_DIR="${TOP_LEVEL}/.local/charts" +HELM_PARALLELISM=${HELM_PARALLELISM:-1} . "$DIR/helm_overrides.sh" ${DIR}/integration-cleanup.sh @@ -20,9 +21,8 @@ ${DIR}/integration-cleanup.sh # (e.g. cassandra from underneath databases-ephemeral) echo "updating recursive dependencies ..." charts=(fake-aws databases-ephemeral redis-cluster wire-server nginx-ingress-controller nginx-ingress-services) -for chart in "${charts[@]}"; do - "$DIR/update.sh" "$CHARTS_DIR/$chart" -done +mkdir -p ~/.parallel && touch ~/.parallel/will-cite +printf '%s\n' "${charts[@]}" | parallel -P "${HELM_PARALLELISM}" "$DIR/update.sh" "$CHARTS_DIR/{}" # FUTUREWORK: use helm functions instead, see https://wearezeta.atlassian.net/browse/SQPIT-723 echo "Generating self-signed certificates..." diff --git a/hack/bin/integration-setup.sh b/hack/bin/integration-setup.sh index 634cc3a49f..59cf0e4f84 100755 --- a/hack/bin/integration-setup.sh +++ b/hack/bin/integration-setup.sh @@ -7,6 +7,7 @@ TOP_LEVEL="$DIR/../.." export NAMESPACE=${NAMESPACE:-test-integration} HELMFILE_ENV=${HELMFILE_ENV:-default} CHARTS_DIR="${TOP_LEVEL}/.local/charts" +HELM_PARALLELISM=${HELM_PARALLELISM:-1} . "$DIR/helm_overrides.sh" @@ -14,9 +15,8 @@ CHARTS_DIR="${TOP_LEVEL}/.local/charts" echo "updating recursive dependencies ..." charts=(fake-aws databases-ephemeral redis-cluster wire-server nginx-ingress-controller nginx-ingress-services) -for chart in "${charts[@]}"; do - "$DIR/update.sh" "$CHARTS_DIR/$chart" -done +mkdir -p ~/.parallel && touch ~/.parallel/will-cite +printf '%s\n' "${charts[@]}" | parallel -P "${HELM_PARALLELISM}" "$DIR/update.sh" "$CHARTS_DIR/{}" echo "Generating self-signed certificates..." export FEDERATION_DOMAIN_BASE="$NAMESPACE.svc.cluster.local" diff --git a/hack/bin/integration-test.sh b/hack/bin/integration-test.sh index 8ffb21d8c7..47c5db9c3d 100755 --- a/hack/bin/integration-test.sh +++ b/hack/bin/integration-test.sh @@ -1,9 +1,93 @@ #!/usr/bin/env bash set -euo pipefail +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" NAMESPACE=${NAMESPACE:-test-integration} +# set to 1 to disable running helm tests in parallel +HELM_PARALLELISM=${HELM_PARALLELISM:-1} +CLEANUP_LOCAL_FILES=${CLEANUP_LOCAL_FILES:-1} # set to 0 to keep files -echo "Running integration tests on wire-server" +echo "Running integration tests on wire-server with parallelism=${HELM_PARALLELISM} ..." CHART=wire-server -helm test --logs -n "${NAMESPACE}" "${NAMESPACE}-${CHART}" --timeout 900s +tests=(galley cargohold gundeck federator spar brig) + +cleanup() { + if (( CLEANUP_LOCAL_FILES > 0 )); then + for t in "${tests[@]}"; do + rm -f "stat-$t" + rm -f "logs-$t" + done + fi +} + +summary() { + echo "===============" + echo "=== summary ===" + echo "===============" + printf '%s\n' "${tests[@]}" | parallel echo "=== tail {}: ===" ';' tail -2 logs-{} + + for t in "${tests[@]}"; do + x=$(cat "stat-$t") + if ((x > 0)); then + echo "$t-integration FAILED ❌. pfff..." + else + echo "$t-integration passed ✅." + fi + done +} + +# Run tests in parallel using GNU parallel (see https://www.gnu.org/software/parallel/) +# The below commands are a little convoluted, but we wish to: +# - run integration tests. If they fail, keep track of this, but still go and get logs, so we see what failed +# - run all tests. Perhaps multiple flaky tests in multiple services exist, if so, we wish to see all problems +mkdir -p ~/.parallel && touch ~/.parallel/will-cite +printf '%s\n' "${tests[@]}" | parallel echo "Running helm tests for {}..." +printf '%s\n' "${tests[@]}" | parallel -P "${HELM_PARALLELISM}" \ + helm test -n "${NAMESPACE}" "${NAMESPACE}-${CHART}" --timeout 900s --filter name="${NAMESPACE}-${CHART}-{}-integration" '> logs-{};' \ + echo '$? > stat-{};' \ + echo "==== Done testing {}. ====" '};' \ + kubectl -n "${NAMESPACE}" logs "${NAMESPACE}-${CHART}-{}-integration" '>> logs-{};' + +summary + +# in case any integration test suite failed, exit this script with an error. +exit_code=0 +for t in "${tests[@]}"; do + x=$(cat "stat-$t") + if ((x > 0)); then + exit_code=1 + fi +done + +if ((exit_code > 0)); then + echo "=======================" + echo "=== failed job logs ===" + echo "=======================" + # in case a integration test suite failed, print relevant logs + for t in "${tests[@]}"; do + x=$(cat "stat-$t") + if ((x > 0)); then + echo "=== logs for failed $t-integration ===" + cat "logs-$t" + fi + done + summary + for t in "${tests[@]}"; do + x=$(cat "stat-$t") + if ((x > 0)); then + echo "=== (relevant) logs for failed $t-integration ===" + "$DIR/integration-logs-relevant-bits.sh" < "logs-$t" + fi + done + summary +fi + +cleanup + +if ((exit_code > 0)); then + echo "Tests failed." + exit 1 +else + echo "All integration tests passed ✅." +fi diff --git a/nix/wire-server.nix b/nix/wire-server.nix index 33320d327d..c215298f8b 100644 --- a/nix/wire-server.nix +++ b/nix/wire-server.nix @@ -298,6 +298,8 @@ let pkgs.cabal2nix pkgs.gnumake pkgs.gnused + pkgs.parallel + pkgs.ripgrep pkgs.helm pkgs.helmfile pkgs.hlint