Improve helm test output; and provide the means (even if disabled due to flaky tests) for parallel helm test executions.

Makefile

@@ -20,6 +20,7 @@ elasticsearch-ephemeral minio-external cassandra-external						\
 nginx-ingress-controller nginx-ingress-services reaper sftd restund coturn		\
 inbucket k8ssandra-test-cluster
 KIND_CLUSTER_NAME     := wire-server
+HELM_PARALLELISM      ?= 1 # 1 for sequential tests; 6 for all-parallel tests
 
 package ?= all
 EXE_SCHEMA := ./dist/$(package)-schema
@@ -315,15 +316,15 @@ kube-integration:  kube-integration-setup kube-integration-test
 
 .PHONY: kube-integration-setup
 kube-integration-setup: charts-integration
-	export NAMESPACE=$(NAMESPACE); ./hack/bin/integration-setup-federation.sh
+	export NAMESPACE=$(NAMESPACE); export HELM_PARALLELISM=$(HELM_PARALLELISM); ./hack/bin/integration-setup-federation.sh
 
 .PHONY: kube-integration-test
 kube-integration-test:
-	export NAMESPACE=$(NAMESPACE); ./hack/bin/integration-test.sh
+	export NAMESPACE=$(NAMESPACE); export HELM_PARALLELISM=$(HELM_PARALLELISM); ./hack/bin/integration-test.sh
 
 .PHONY: kube-integration-teardown
 kube-integration-teardown:
-	export NAMESPACE=$(NAMESPACE); ./hack/bin/integration-teardown-federation.sh
+	export NAMESPACE=$(NAMESPACE); export HELM_PARALLELISM=$(HELM_PARALLELISM); ./hack/bin/integration-teardown-federation.sh
 
 .PHONY: kube-integration-e2e-telepresence
 kube-integration-e2e-telepresence:

changelog.d/5-internal/parallel-helm-tests

@@ -0,0 +1 @@
+Add config to allow to run helm tests for different services in parallel; improve integration test output logs

hack/bin/integration-logs-relevant-bits.sh

@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+USAGE="
+Filter out garbage from logs (but keep colors and highlight problems).
+
+(Adapted from logfilter.sh by Stefan Matting)
+
+To use it pipe log output from integration-test.sh into this tool.
+
+Usage: logs | $0 [options]
+"
+
+red_color="\x1b\[;1m\x1b\[31m"
+problem_markers="Failures:|FAILED|ExitFailure""\
+|\^+""\
+|FAIL""\
+|tests failed""\
+|Test suite failure""\
+|""$red_color""error:""\
+|""$red_color""\
+|Test suite .+ failed"
+
+exit_usage() {
+    echo "$USAGE"
+    exit 1
+}
+
+# remove debug/info logs
+# often this is just noise like connection to cassandra.
+excludeLogEntries() {
+    grep -v '^{".*Info"' |
+        grep -v '^{".*Debug"' |
+        grep -v '^20.*, D, .*socket: [0-9]\+>$'
+}
+
+cleanup() {
+    # replace backspaces with newlines
+    # remove "Progress" lines
+    # Remove blank lines
+    # add newline between interleaved test name and log output lines
+    sed 's/\x08\+/\n/g' |
+        sed '/^Progress [0-9]\+/d' |
+        sed '/^\s\+$/d' |
+        sed 's/:\s\+{/:\n{/g'
+}
+
+grepper() {
+    # print 10 lines before/after for context
+    rg "$problem_markers" --color=always -A 10 -B 10
+    echo -e "\033[0m"
+}
+
+cleanup | excludeLogEntries | grepper

hack/bin/integration-setup-federation.sh

@@ -7,6 +7,7 @@ TOP_LEVEL="$DIR/../.."
 export NAMESPACE=${NAMESPACE:-test-integration}
 HELMFILE_ENV=${HELMFILE_ENV:-default}
 CHARTS_DIR="${TOP_LEVEL}/.local/charts"
+HELM_PARALLELISM=${HELM_PARALLELISM:-1}
 
 . "$DIR/helm_overrides.sh"
 ${DIR}/integration-cleanup.sh
@@ -20,9 +21,8 @@ ${DIR}/integration-cleanup.sh
 # (e.g. cassandra from underneath databases-ephemeral)
 echo "updating recursive dependencies ..."
 charts=(fake-aws databases-ephemeral redis-cluster wire-server nginx-ingress-controller nginx-ingress-services)
-for chart in "${charts[@]}"; do
-    "$DIR/update.sh" "$CHARTS_DIR/$chart"
-done
+mkdir -p ~/.parallel && touch ~/.parallel/will-cite
+printf '%s\n' "${charts[@]}" | parallel -P "${HELM_PARALLELISM}" "$DIR/update.sh" "$CHARTS_DIR/{}"
 
 # FUTUREWORK: use helm functions instead, see https://wearezeta.atlassian.net/browse/SQPIT-723
 echo "Generating self-signed certificates..."

hack/bin/integration-setup.sh

@@ -7,16 +7,16 @@ TOP_LEVEL="$DIR/../.."
 export NAMESPACE=${NAMESPACE:-test-integration}
 HELMFILE_ENV=${HELMFILE_ENV:-default}
 CHARTS_DIR="${TOP_LEVEL}/.local/charts"
+HELM_PARALLELISM=${HELM_PARALLELISM:-1}
 
 . "$DIR/helm_overrides.sh"
 
 "${DIR}/integration-cleanup.sh"
 
 echo "updating recursive dependencies ..."
 charts=(fake-aws databases-ephemeral redis-cluster wire-server nginx-ingress-controller nginx-ingress-services)
-for chart in "${charts[@]}"; do
-    "$DIR/update.sh" "$CHARTS_DIR/$chart"
-done
+mkdir -p ~/.parallel && touch ~/.parallel/will-cite
+printf '%s\n' "${charts[@]}" | parallel -P "${HELM_PARALLELISM}" "$DIR/update.sh" "$CHARTS_DIR/{}"
 
 echo "Generating self-signed certificates..."
 export FEDERATION_DOMAIN_BASE="$NAMESPACE.svc.cluster.local"

hack/bin/integration-test.sh

@@ -1,9 +1,93 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 NAMESPACE=${NAMESPACE:-test-integration}
+# set to 1 to disable running helm tests in parallel
+HELM_PARALLELISM=${HELM_PARALLELISM:-1}
+CLEANUP_LOCAL_FILES=${CLEANUP_LOCAL_FILES:-1} # set to 0 to keep files
 
-echo "Running integration tests on wire-server"
+echo "Running integration tests on wire-server with parallelism=${HELM_PARALLELISM} ..."
 
 CHART=wire-server
-helm test --logs -n "${NAMESPACE}" "${NAMESPACE}-${CHART}" --timeout 900s
+tests=(galley cargohold gundeck federator spar brig)
+
+cleanup() {
+    if (( CLEANUP_LOCAL_FILES > 0 )); then
+        for t in "${tests[@]}"; do
+            rm -f "stat-$t"
+            rm -f "logs-$t"
+        done
+    fi
+}
+
+summary() {
+    echo "==============="
+    echo "=== summary ==="
+    echo "==============="
+    printf '%s\n' "${tests[@]}" | parallel echo "=== tail {}: ===" ';' tail -2 logs-{}
+
+    for t in "${tests[@]}"; do
+        x=$(cat "stat-$t")
+        if ((x > 0)); then
+            echo "$t-integration FAILED ❌. pfff..."
+        else
+            echo "$t-integration passed ✅."
+        fi
+    done
+}
+
+# Run tests in parallel using GNU parallel (see https://www.gnu.org/software/parallel/)
+# The below commands are a little convoluted, but we wish to:
+# - run integration tests. If they fail, keep track of this, but still go and get logs, so we see what failed
+# - run all tests. Perhaps multiple flaky tests in multiple services exist, if so, we wish to see all problems
+mkdir -p ~/.parallel && touch ~/.parallel/will-cite
+printf '%s\n' "${tests[@]}" | parallel echo "Running helm tests for {}..."
+printf '%s\n' "${tests[@]}" | parallel -P "${HELM_PARALLELISM}" \
+    helm test -n "${NAMESPACE}" "${NAMESPACE}-${CHART}" --timeout 900s --filter name="${NAMESPACE}-${CHART}-{}-integration" '> logs-{};' \
+    echo '$? > stat-{};' \
+    echo "==== Done testing {}. ====" '};' \
+    kubectl -n "${NAMESPACE}" logs "${NAMESPACE}-${CHART}-{}-integration" '>> logs-{};'
+
+summary
+
+# in case any integration test suite failed, exit this script with an error.
+exit_code=0
+for t in "${tests[@]}"; do
+    x=$(cat "stat-$t")
+    if ((x > 0)); then
+        exit_code=1
+    fi
+done
+
+if ((exit_code > 0)); then
+    echo "======================="
+    echo "=== failed job logs ==="
+    echo "======================="
+    # in case a integration test suite failed, print relevant logs
+    for t in "${tests[@]}"; do
+        x=$(cat "stat-$t")
+        if ((x > 0)); then
+            echo "=== logs for failed $t-integration ==="
+            cat "logs-$t"
+        fi
+    done
+    summary
+    for t in "${tests[@]}"; do
+        x=$(cat "stat-$t")
+        if ((x > 0)); then
+            echo "=== (relevant) logs for failed $t-integration ==="
+            "$DIR/integration-logs-relevant-bits.sh" < "logs-$t"
+        fi
+    done
+    summary
+fi
+
+cleanup
+
+if ((exit_code > 0)); then
+    echo "Tests failed."
+    exit 1
+else
+    echo "All integration tests passed ✅."
+fi

nix/wire-server.nix

@@ -298,6 +298,8 @@ let
     pkgs.cabal2nix
     pkgs.gnumake
     pkgs.gnused
+    pkgs.parallel
+    pkgs.ripgrep
     pkgs.helm
     pkgs.helmfile
     pkgs.hlint