From c61225585ed22ae7590ebf90df21005f727baafb Mon Sep 17 00:00:00 2001
From: jschaul <jschaul@users.noreply.github.com>
Date: Mon, 30 Jan 2023 17:16:05 +0100
Subject: [PATCH 01/24] add the 'parallel' tool to nix for easier process
 spawning in bash

---
 nix/wire-server.nix | 1 +
 1 file changed, 1 insertion(+)

diff --git a/nix/wire-server.nix b/nix/wire-server.nix
index 33320d327d..c7b7eb2fbf 100644
--- a/nix/wire-server.nix
+++ b/nix/wire-server.nix
@@ -298,6 +298,7 @@ let
     pkgs.cabal2nix
     pkgs.gnumake
     pkgs.gnused
+    pkgs.parallel
     pkgs.helm
     pkgs.helmfile
     pkgs.hlint

From f7aef325df4880e4e1143bf64a1d858c73e0c659 Mon Sep 17 00:00:00 2001
From: jschaul <jschaul@users.noreply.github.com>
Date: Mon, 30 Jan 2023 17:15:16 +0100
Subject: [PATCH 02/24] parallelize helm dependency updates

---
 hack/bin/integration-setup-federation.sh | 4 +---
 hack/bin/integration-setup.sh            | 4 +---
 2 files changed, 2 insertions(+), 6 deletions(-)

diff --git a/hack/bin/integration-setup-federation.sh b/hack/bin/integration-setup-federation.sh
index 7795226bab..af4d68fc90 100755
--- a/hack/bin/integration-setup-federation.sh
+++ b/hack/bin/integration-setup-federation.sh
@@ -20,9 +20,7 @@ ${DIR}/integration-cleanup.sh
 # (e.g. cassandra from underneath databases-ephemeral)
 echo "updating recursive dependencies ..."
 charts=(fake-aws databases-ephemeral redis-cluster wire-server nginx-ingress-controller nginx-ingress-services)
-for chart in "${charts[@]}"; do
-    "$DIR/update.sh" "$CHARTS_DIR/$chart"
-done
+printf '%s\n' "${charts[@]}" | parallel "$DIR/update.sh" "$CHARTS_DIR/{}"
 
 # FUTUREWORK: use helm functions instead, see https://wearezeta.atlassian.net/browse/SQPIT-723
 echo "Generating self-signed certificates..."
diff --git a/hack/bin/integration-setup.sh b/hack/bin/integration-setup.sh
index 634cc3a49f..8600b1ac36 100755
--- a/hack/bin/integration-setup.sh
+++ b/hack/bin/integration-setup.sh
@@ -14,9 +14,7 @@ CHARTS_DIR="${TOP_LEVEL}/.local/charts"
 
 echo "updating recursive dependencies ..."
 charts=(fake-aws databases-ephemeral redis-cluster wire-server nginx-ingress-controller nginx-ingress-services)
-for chart in "${charts[@]}"; do
-    "$DIR/update.sh" "$CHARTS_DIR/$chart"
-done
+printf '%s\n' "${charts[@]}" | parallel "$DIR/update.sh" "$CHARTS_DIR/{}"
 
 echo "Generating self-signed certificates..."
 export FEDERATION_DOMAIN_BASE="$NAMESPACE.svc.cluster.local"

From f76f3990fb05e5772d5a1ffd9f305dead07fe15a Mon Sep 17 00:00:00 2001
From: jschaul <jschaul@users.noreply.github.com>
Date: Mon, 30 Jan 2023 17:48:09 +0100
Subject: [PATCH 03/24] run helm tests in parallel

---
 hack/bin/integration-test.sh | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/hack/bin/integration-test.sh b/hack/bin/integration-test.sh
index 8ffb21d8c7..417e0d7806 100755
--- a/hack/bin/integration-test.sh
+++ b/hack/bin/integration-test.sh
@@ -6,4 +6,11 @@ NAMESPACE=${NAMESPACE:-test-integration}
 echo "Running integration tests on wire-server"
 
 CHART=wire-server
-helm test --logs -n "${NAMESPACE}" "${NAMESPACE}-${CHART}" --timeout 900s
+tests=(cargohold gundeck spar brig galley federator)
+# Run tests in parallel using GNU parallel (see https://www.gnu.org/software/parallel/)
+printf '%s\n' "${tests[@]}" | parallel echo "Running helm tests for {}..."
+printf '%s\n' "${tests[@]}" | parallel \
+    helm test -n "${NAMESPACE}" "${NAMESPACE}-${CHART}" --timeout 900s --filter name="${NAMESPACE}-${CHART}-{}-integration" '| grep -v NOTES;' \
+    echo "==== Done testing {}. Here are logs: ====" ';' \
+    kubectl -n "${NAMESPACE}" logs "${NAMESPACE}-${CHART}-{}-integration" ';' \
+    echo "==== Above logs are for {}.===="

From 7ba7c3141dd2a3d61a118cb256e54ac4fbd3536a Mon Sep 17 00:00:00 2001
From: jschaul <jschaul@users.noreply.github.com>
Date: Mon, 30 Jan 2023 17:53:26 +0100
Subject: [PATCH 04/24] changelog

---
 changelog.d/5-internal/parallel-helm-tests | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/5-internal/parallel-helm-tests

diff --git a/changelog.d/5-internal/parallel-helm-tests b/changelog.d/5-internal/parallel-helm-tests
new file mode 100644
index 0000000000..38e95a823a
--- /dev/null
+++ b/changelog.d/5-internal/parallel-helm-tests
@@ -0,0 +1 @@
+Run helm tests for different services in parallel

From e29ecebf8888c987e67e026d2264453d24ff5395 Mon Sep 17 00:00:00 2001
From: jschaul <jschaul@users.noreply.github.com>
Date: Mon, 30 Jan 2023 19:06:13 +0100
Subject: [PATCH 05/24] exit code handling

---
 hack/bin/integration-test.sh | 25 ++++++++++++++++++++-----
 1 file changed, 20 insertions(+), 5 deletions(-)

diff --git a/hack/bin/integration-test.sh b/hack/bin/integration-test.sh
index 417e0d7806..d7dc583939 100755
--- a/hack/bin/integration-test.sh
+++ b/hack/bin/integration-test.sh
@@ -8,9 +8,24 @@ echo "Running integration tests on wire-server"
 CHART=wire-server
 tests=(cargohold gundeck spar brig galley federator)
 # Run tests in parallel using GNU parallel (see https://www.gnu.org/software/parallel/)
+# The below commands are a little convoluted, but we wish to:
+# - run integration tests. If they fail, keep track of this, but still go and get logs, so we see what failed
+# - run all tests. Perhaps multiple flaky tests in multiple services exist, if so, we wish to see all problems
+# --halt now,fail=1
 printf '%s\n' "${tests[@]}" | parallel echo "Running helm tests for {}..."
-printf '%s\n' "${tests[@]}" | parallel \
-    helm test -n "${NAMESPACE}" "${NAMESPACE}-${CHART}" --timeout 900s --filter name="${NAMESPACE}-${CHART}-{}-integration" '| grep -v NOTES;' \
-    echo "==== Done testing {}. Here are logs: ====" ';' \
-    kubectl -n "${NAMESPACE}" logs "${NAMESPACE}-${CHART}-{}-integration" ';' \
-    echo "==== Above logs are for {}.===="
+printf '%s\n' "${tests[@]}" | parallel -P 6 --results results.csv \
+    helm test -n "${NAMESPACE}" "${NAMESPACE}-${CHART}" --timeout 900s --filter name="${NAMESPACE}-${CHART}-{}-integration" '> {};' \
+    echo '$? > stat-{};' \
+    echo "==== Done testing {}. Here are logs: ====" '>> {};' \
+    kubectl -n "${NAMESPACE}" logs "${NAMESPACE}-${CHART}-{}-integration" '>> {};' \
+    echo "==== Above logs are for {}.====" '>> {};'
+printf '%s\n' "${tests[@]}" | parallel echo "=== tail {}: ===" ';' tail -3 {}
+
+# in case any integration test suite failed, exit this script with an error.
+for res in "${tests[@]}"; do
+    x=$(cat "stat-$res")
+    if ((x > 0)); then
+        echo "$res FAILED. pfff..."
+        exit 1
+    fi
+done

From 3a72191730b5b3c1d56efd69eb65b1d8f42f1e59 Mon Sep 17 00:00:00 2001
From: jschaul <jschaul@users.noreply.github.com>
Date: Mon, 30 Jan 2023 19:10:02 +0100
Subject: [PATCH 06/24] make output a bit nicer

---
 hack/bin/integration-test.sh | 24 +++++++++++++++++++-----
 1 file changed, 19 insertions(+), 5 deletions(-)

diff --git a/hack/bin/integration-test.sh b/hack/bin/integration-test.sh
index d7dc583939..4f5e216d16 100755
--- a/hack/bin/integration-test.sh
+++ b/hack/bin/integration-test.sh
@@ -17,15 +17,29 @@ printf '%s\n' "${tests[@]}" | parallel -P 6 --results results.csv \
     helm test -n "${NAMESPACE}" "${NAMESPACE}-${CHART}" --timeout 900s --filter name="${NAMESPACE}-${CHART}-{}-integration" '> {};' \
     echo '$? > stat-{};' \
     echo "==== Done testing {}. Here are logs: ====" '>> {};' \
-    kubectl -n "${NAMESPACE}" logs "${NAMESPACE}-${CHART}-{}-integration" '>> {};' \
-    echo "==== Above logs are for {}.====" '>> {};'
+    kubectl -n "${NAMESPACE}" logs "${NAMESPACE}-${CHART}-{}-integration" '>> {};'
+
+echo "==============="
+echo "=== summary ==="
+echo "==============="
 printf '%s\n' "${tests[@]}" | parallel echo "=== tail {}: ===" ';' tail -3 {}
 
+echo "======================="
+echo "=== failed job logs ==="
+echo "======================="
+# in case a integration test suite failed, print relevant logs
+for t in "${tests[@]}"; do
+    x=$(cat "stat-$t")
+    if ((x > 0)); then
+        cat "$t"
+    fi
+done
+
 # in case any integration test suite failed, exit this script with an error.
-for res in "${tests[@]}"; do
-    x=$(cat "stat-$res")
+for t in "${tests[@]}"; do
+    x=$(cat "stat-$t")
     if ((x > 0)); then
-        echo "$res FAILED. pfff..."
+        echo "$t FAILED. pfff..."
         exit 1
     fi
 done

From 60af457fa69908aea47d1564b6c886bf91192079 Mon Sep 17 00:00:00 2001
From: jschaul <jschaul@users.noreply.github.com>
Date: Tue, 31 Jan 2023 12:02:18 +0100
Subject: [PATCH 07/24] rework output

---
 hack/bin/integration-test.sh | 72 ++++++++++++++++++++++++------------
 1 file changed, 48 insertions(+), 24 deletions(-)

diff --git a/hack/bin/integration-test.sh b/hack/bin/integration-test.sh
index 4f5e216d16..4150af9bc3 100755
--- a/hack/bin/integration-test.sh
+++ b/hack/bin/integration-test.sh
@@ -6,40 +6,64 @@ NAMESPACE=${NAMESPACE:-test-integration}
 echo "Running integration tests on wire-server"
 
 CHART=wire-server
-tests=(cargohold gundeck spar brig galley federator)
+tests=(cargohold gundeck federator spar brig galley)
+
+summary() {
+    echo "==============="
+    echo "=== summary ==="
+    echo "==============="
+    printf '%s\n' "${tests[@]}" | parallel echo "=== tail {}: ===" ';' tail -2 logs-{}
+
+    for t in "${tests[@]}"; do
+        x=$(cat "stat-$t")
+        if ((x > 0)); then
+            echo "$t-integration FAILED ❌. pfff..."
+        else
+            echo "$t-integration passed ✅."
+        fi
+    done
+}
+
 # Run tests in parallel using GNU parallel (see https://www.gnu.org/software/parallel/)
 # The below commands are a little convoluted, but we wish to:
 # - run integration tests. If they fail, keep track of this, but still go and get logs, so we see what failed
 # - run all tests. Perhaps multiple flaky tests in multiple services exist, if so, we wish to see all problems
-# --halt now,fail=1
 printf '%s\n' "${tests[@]}" | parallel echo "Running helm tests for {}..."
-printf '%s\n' "${tests[@]}" | parallel -P 6 --results results.csv \
-    helm test -n "${NAMESPACE}" "${NAMESPACE}-${CHART}" --timeout 900s --filter name="${NAMESPACE}-${CHART}-{}-integration" '> {};' \
+printf '%s\n' "${tests[@]}" | parallel -P 6 \
+    helm test -n "${NAMESPACE}" "${NAMESPACE}-${CHART}" --timeout 900s --filter name="${NAMESPACE}-${CHART}-{}-integration" '> logs-{};' \
     echo '$? > stat-{};' \
-    echo "==== Done testing {}. Here are logs: ====" '>> {};' \
-    kubectl -n "${NAMESPACE}" logs "${NAMESPACE}-${CHART}-{}-integration" '>> {};'
-
-echo "==============="
-echo "=== summary ==="
-echo "==============="
-printf '%s\n' "${tests[@]}" | parallel echo "=== tail {}: ===" ';' tail -3 {}
-
-echo "======================="
-echo "=== failed job logs ==="
-echo "======================="
-# in case a integration test suite failed, print relevant logs
-for t in "${tests[@]}"; do
-    x=$(cat "stat-$t")
-    if ((x > 0)); then
-        cat "$t"
-    fi
-done
+    echo "==== Done testing {}. ====" '};' \
+    kubectl -n "${NAMESPACE}" logs "${NAMESPACE}-${CHART}-{}-integration" '>> logs-{};'
+
+summary
 
 # in case any integration test suite failed, exit this script with an error.
+exit_code=0
 for t in "${tests[@]}"; do
     x=$(cat "stat-$t")
     if ((x > 0)); then
-        echo "$t FAILED. pfff..."
-        exit 1
+        exit_code=1
     fi
 done
+
+if ((exit_code > 0)); then
+    echo "======================="
+    echo "=== failed job logs ==="
+    echo "======================="
+    # in case a integration test suite failed, print relevant logs
+    for t in "${tests[@]}"; do
+        x=$(cat "stat-$t")
+        if ((x > 0)); then
+            echo "=== logs for failed $t-integration ==="
+            cat "logs-$t"
+        fi
+    done
+    summary
+fi
+
+if ((exit_code > 0)); then
+    echo "Tests failed."
+    exit 1
+else
+    echo "All integration tests passed ✅."
+fi

From d20aff94348afe10b64d98c19efff92bc5fb92a3 Mon Sep 17 00:00:00 2001
From: jschaul <jschaul@users.noreply.github.com>
Date: Tue, 31 Jan 2023 12:51:22 +0100
Subject: [PATCH 08/24] provide summary and failed-test logs at the bottom of
 the output

---
 hack/bin/integration-logs-relevant-bits.sh | 43 ++++++++++++++++++++++
 hack/bin/integration-test.sh               |  9 +++++
 2 files changed, 52 insertions(+)
 create mode 100755 hack/bin/integration-logs-relevant-bits.sh

diff --git a/hack/bin/integration-logs-relevant-bits.sh b/hack/bin/integration-logs-relevant-bits.sh
new file mode 100755
index 0000000000..a1c7656208
--- /dev/null
+++ b/hack/bin/integration-logs-relevant-bits.sh
@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+USAGE="
+Filter out garbage from logs (but keep colors and highlight problems).
+
+(Adapted from logfilter.sh by Stefan Matting)
+
+To use it pipe log output from integration-test.sh into this tool.
+
+Usage: logs | $0 [options]
+"
+
+red_color="\x1b\[;1m\x1b\[31m"
+problem_markers="Failures:|FAILED|ExitFailure""\
+|\^+""\
+|FAIL""\
+|tests failed""\
+|Test suite failure""\
+|""$red_color""error:""\
+|""$red_color""\
+|Test suite .+ failed"
+
+exit_usage() {
+    echo "$USAGE"
+    exit 1
+}
+
+cleanup() {
+    # replace backspaces with newlines
+    # remove "Progress" lines
+    # Remove blank lines
+    sed 's/\x08\+/\n/g' |
+        sed '/^Progress [0-9]\+/d' |
+        sed '/^\s\+$/d'
+}
+
+grepper() {
+    # print 10 lines before/after for context
+    rg "$problem_markers" --color=always -A 10 -B 10
+}
+
+cleanup | grepper
diff --git a/hack/bin/integration-test.sh b/hack/bin/integration-test.sh
index 4150af9bc3..9546175ae9 100755
--- a/hack/bin/integration-test.sh
+++ b/hack/bin/integration-test.sh
@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 NAMESPACE=${NAMESPACE:-test-integration}
 
 echo "Running integration tests on wire-server"
@@ -59,6 +60,14 @@ if ((exit_code > 0)); then
         fi
     done
     summary
+    for t in "${tests[@]}"; do
+        x=$(cat "stat-$t")
+        if ((x > 0)); then
+            echo "=== (relevant) logs for failed $t-integration ==="
+            grep -v '"level"' "logs-$t" | "$DIR/integration-logs-relevant-bits.sh"
+        fi
+    done
+    summary
 fi
 
 if ((exit_code > 0)); then

From bbe80de6d31887123fce4194723856072abd6751 Mon Sep 17 00:00:00 2001
From: jschaul <jschaul@users.noreply.github.com>
Date: Tue, 31 Jan 2023 12:53:39 +0100
Subject: [PATCH 09/24] nix: add ripgrep

---
 nix/wire-server.nix | 1 +
 1 file changed, 1 insertion(+)

diff --git a/nix/wire-server.nix b/nix/wire-server.nix
index c7b7eb2fbf..30dd52a4de 100644
--- a/nix/wire-server.nix
+++ b/nix/wire-server.nix
@@ -375,6 +375,7 @@ in
       pkgs.rsync
       pkgs.wget
       pkgs.yq
+      pkgs.ripgrep
       pkgs.nginz
 
       pkgs.cabal-install

From c6e5bb96879e269b5523dbd46b9b3b6e32a29451 Mon Sep 17 00:00:00 2001
From: jschaul <jschaul@users.noreply.github.com>
Date: Tue, 31 Jan 2023 13:11:42 +0100
Subject: [PATCH 10/24] disable citation warning

---
 hack/bin/integration-setup-federation.sh | 1 +
 hack/bin/integration-setup.sh            | 1 +
 hack/bin/integration-test.sh             | 1 +
 3 files changed, 3 insertions(+)

diff --git a/hack/bin/integration-setup-federation.sh b/hack/bin/integration-setup-federation.sh
index af4d68fc90..9f366e345f 100755
--- a/hack/bin/integration-setup-federation.sh
+++ b/hack/bin/integration-setup-federation.sh
@@ -20,6 +20,7 @@ ${DIR}/integration-cleanup.sh
 # (e.g. cassandra from underneath databases-ephemeral)
 echo "updating recursive dependencies ..."
 charts=(fake-aws databases-ephemeral redis-cluster wire-server nginx-ingress-controller nginx-ingress-services)
+touch ~/.parallel/will-cite
 printf '%s\n' "${charts[@]}" | parallel "$DIR/update.sh" "$CHARTS_DIR/{}"
 
 # FUTUREWORK: use helm functions instead, see https://wearezeta.atlassian.net/browse/SQPIT-723
diff --git a/hack/bin/integration-setup.sh b/hack/bin/integration-setup.sh
index 8600b1ac36..6b5d4f1e14 100755
--- a/hack/bin/integration-setup.sh
+++ b/hack/bin/integration-setup.sh
@@ -14,6 +14,7 @@ CHARTS_DIR="${TOP_LEVEL}/.local/charts"
 
 echo "updating recursive dependencies ..."
 charts=(fake-aws databases-ephemeral redis-cluster wire-server nginx-ingress-controller nginx-ingress-services)
+touch ~/.parallel/will-cite
 printf '%s\n' "${charts[@]}" | parallel "$DIR/update.sh" "$CHARTS_DIR/{}"
 
 echo "Generating self-signed certificates..."
diff --git a/hack/bin/integration-test.sh b/hack/bin/integration-test.sh
index 9546175ae9..208b4ad6b9 100755
--- a/hack/bin/integration-test.sh
+++ b/hack/bin/integration-test.sh
@@ -29,6 +29,7 @@ summary() {
 # The below commands are a little convoluted, but we wish to:
 # - run integration tests. If they fail, keep track of this, but still go and get logs, so we see what failed
 # - run all tests. Perhaps multiple flaky tests in multiple services exist, if so, we wish to see all problems
+touch ~/.parallel/will-cite
 printf '%s\n' "${tests[@]}" | parallel echo "Running helm tests for {}..."
 printf '%s\n' "${tests[@]}" | parallel -P 6 \
     helm test -n "${NAMESPACE}" "${NAMESPACE}-${CHART}" --timeout 900s --filter name="${NAMESPACE}-${CHART}-{}-integration" '> logs-{};' \

From b58116de2b4680ff9d5a35845bb44a9254da0645 Mon Sep 17 00:00:00 2001
From: jschaul <jschaul@users.noreply.github.com>
Date: Tue, 31 Jan 2023 17:04:43 +0100
Subject: [PATCH 11/24] include ripgrep in CI image

---
 nix/wire-server.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/nix/wire-server.nix b/nix/wire-server.nix
index 30dd52a4de..c215298f8b 100644
--- a/nix/wire-server.nix
+++ b/nix/wire-server.nix
@@ -299,6 +299,7 @@ let
     pkgs.gnumake
     pkgs.gnused
     pkgs.parallel
+    pkgs.ripgrep
     pkgs.helm
     pkgs.helmfile
     pkgs.hlint
@@ -375,7 +376,6 @@ in
       pkgs.rsync
       pkgs.wget
       pkgs.yq
-      pkgs.ripgrep
       pkgs.nginz
 
       pkgs.cabal-install

From a7d23802a7636e8d8ef56e6afbd0615912195e78 Mon Sep 17 00:00:00 2001
From: jschaul <jschaul@users.noreply.github.com>
Date: Tue, 31 Jan 2023 17:39:15 +0100
Subject: [PATCH 12/24] improve relevant-output-hunt script

---
 hack/bin/integration-logs-relevant-bits.sh | 13 +++++++++++--
 hack/bin/integration-test.sh               |  2 +-
 2 files changed, 12 insertions(+), 3 deletions(-)

diff --git a/hack/bin/integration-logs-relevant-bits.sh b/hack/bin/integration-logs-relevant-bits.sh
index a1c7656208..4ebc9f6c35 100755
--- a/hack/bin/integration-logs-relevant-bits.sh
+++ b/hack/bin/integration-logs-relevant-bits.sh
@@ -26,13 +26,22 @@ exit_usage() {
     exit 1
 }
 
+# remove debug/info logs
+# often this is just noise like connection to cassandra.
+excludeLogEntries() {
+    grep -v '^{".*Info"' |
+        grep -v '^{".*Debug"'
+}
+
 cleanup() {
     # replace backspaces with newlines
     # remove "Progress" lines
     # Remove blank lines
+    # add newline between interleaved test name and log output lines
     sed 's/\x08\+/\n/g' |
         sed '/^Progress [0-9]\+/d' |
-        sed '/^\s\+$/d'
+        sed '/^\s\+$/d' |
+        sed 's/:\s\+{/:\n{/g'
 }
 
 grepper() {
@@ -40,4 +49,4 @@ grepper() {
     rg "$problem_markers" --color=always -A 10 -B 10
 }
 
-cleanup | grepper
+cleanup | excludeLogEntries | grepper
diff --git a/hack/bin/integration-test.sh b/hack/bin/integration-test.sh
index 208b4ad6b9..121abe67e4 100755
--- a/hack/bin/integration-test.sh
+++ b/hack/bin/integration-test.sh
@@ -65,7 +65,7 @@ if ((exit_code > 0)); then
         x=$(cat "stat-$t")
         if ((x > 0)); then
             echo "=== (relevant) logs for failed $t-integration ==="
-            grep -v '"level"' "logs-$t" | "$DIR/integration-logs-relevant-bits.sh"
+            "$DIR/integration-logs-relevant-bits.sh" < "logs-$t"
         fi
     done
     summary

From 22e3357ac54280f7a0373cfb9084e7920255b1d1 Mon Sep 17 00:00:00 2001
From: jschaul <jschaul@users.noreply.github.com>
Date: Tue, 31 Jan 2023 17:58:22 +0100
Subject: [PATCH 13/24] helm parallelism variable

---
 hack/bin/integration-test.sh | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/hack/bin/integration-test.sh b/hack/bin/integration-test.sh
index 121abe67e4..bd608d625d 100755
--- a/hack/bin/integration-test.sh
+++ b/hack/bin/integration-test.sh
@@ -3,6 +3,8 @@ set -euo pipefail
 
 DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 NAMESPACE=${NAMESPACE:-test-integration}
+# set to 1 to disable running helm tests in parallel
+HELM_PARALLELISM=${HELM_PARALLELISM:-6}
 
 echo "Running integration tests on wire-server"
 
@@ -31,7 +33,7 @@ summary() {
 # - run all tests. Perhaps multiple flaky tests in multiple services exist, if so, we wish to see all problems
 touch ~/.parallel/will-cite
 printf '%s\n' "${tests[@]}" | parallel echo "Running helm tests for {}..."
-printf '%s\n' "${tests[@]}" | parallel -P 6 \
+printf '%s\n' "${tests[@]}" | parallel -P "${HELM_PARALLELISM}" \
     helm test -n "${NAMESPACE}" "${NAMESPACE}-${CHART}" --timeout 900s --filter name="${NAMESPACE}-${CHART}-{}-integration" '> logs-{};' \
     echo '$? > stat-{};' \
     echo "==== Done testing {}. ====" '};' \

From 0c68d55d1533e648dc7a07fe0f19868561715a29 Mon Sep 17 00:00:00 2001
From: jschaul <jschaul@users.noreply.github.com>
Date: Tue, 31 Jan 2023 17:59:16 +0100
Subject: [PATCH 14/24] try without parallelism: do tests succeed now?

---
 hack/bin/integration-test.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hack/bin/integration-test.sh b/hack/bin/integration-test.sh
index bd608d625d..c1f7216379 100755
--- a/hack/bin/integration-test.sh
+++ b/hack/bin/integration-test.sh
@@ -4,7 +4,7 @@ set -euo pipefail
 DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 NAMESPACE=${NAMESPACE:-test-integration}
 # set to 1 to disable running helm tests in parallel
-HELM_PARALLELISM=${HELM_PARALLELISM:-6}
+HELM_PARALLELISM=${HELM_PARALLELISM:-1}
 
 echo "Running integration tests on wire-server"
 

From 174de1bd119ba96d9eb21d8af3052d8540079d6d Mon Sep 17 00:00:00 2001
From: jschaul <jschaul@users.noreply.github.com>
Date: Tue, 31 Jan 2023 18:04:09 +0100
Subject: [PATCH 15/24] nonsense with parallel command and citations

---
 hack/bin/integration-setup-federation.sh | 2 +-
 hack/bin/integration-setup.sh            | 2 +-
 hack/bin/integration-test.sh             | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/hack/bin/integration-setup-federation.sh b/hack/bin/integration-setup-federation.sh
index 9f366e345f..39198ca63b 100755
--- a/hack/bin/integration-setup-federation.sh
+++ b/hack/bin/integration-setup-federation.sh
@@ -20,7 +20,7 @@ ${DIR}/integration-cleanup.sh
 # (e.g. cassandra from underneath databases-ephemeral)
 echo "updating recursive dependencies ..."
 charts=(fake-aws databases-ephemeral redis-cluster wire-server nginx-ingress-controller nginx-ingress-services)
-touch ~/.parallel/will-cite
+mkdir -p ~/.parallel && touch ~/.parallel/will-cite
 printf '%s\n' "${charts[@]}" | parallel "$DIR/update.sh" "$CHARTS_DIR/{}"
 
 # FUTUREWORK: use helm functions instead, see https://wearezeta.atlassian.net/browse/SQPIT-723
diff --git a/hack/bin/integration-setup.sh b/hack/bin/integration-setup.sh
index 6b5d4f1e14..91cac6bb5f 100755
--- a/hack/bin/integration-setup.sh
+++ b/hack/bin/integration-setup.sh
@@ -14,7 +14,7 @@ CHARTS_DIR="${TOP_LEVEL}/.local/charts"
 
 echo "updating recursive dependencies ..."
 charts=(fake-aws databases-ephemeral redis-cluster wire-server nginx-ingress-controller nginx-ingress-services)
-touch ~/.parallel/will-cite
+mkdir -p ~/.parallel && touch ~/.parallel/will-cite
 printf '%s\n' "${charts[@]}" | parallel "$DIR/update.sh" "$CHARTS_DIR/{}"
 
 echo "Generating self-signed certificates..."
diff --git a/hack/bin/integration-test.sh b/hack/bin/integration-test.sh
index c1f7216379..8f24c3f000 100755
--- a/hack/bin/integration-test.sh
+++ b/hack/bin/integration-test.sh
@@ -31,7 +31,7 @@ summary() {
 # The below commands are a little convoluted, but we wish to:
 # - run integration tests. If they fail, keep track of this, but still go and get logs, so we see what failed
 # - run all tests. Perhaps multiple flaky tests in multiple services exist, if so, we wish to see all problems
-touch ~/.parallel/will-cite
+mkdir -p ~/.parallel && touch ~/.parallel/will-cite
 printf '%s\n' "${tests[@]}" | parallel echo "Running helm tests for {}..."
 printf '%s\n' "${tests[@]}" | parallel -P "${HELM_PARALLELISM}" \
     helm test -n "${NAMESPACE}" "${NAMESPACE}-${CHART}" --timeout 900s --filter name="${NAMESPACE}-${CHART}-{}-integration" '> logs-{};' \

From 282d6e5798562ebdc8bb9ef3938bf32dfee50635 Mon Sep 17 00:00:00 2001
From: jschaul <jschaul@users.noreply.github.com>
Date: Wed, 1 Feb 2023 16:18:57 +0100
Subject: [PATCH 16/24] set parallelism=1 everywhere for the time being

---
 Makefile                                 | 7 ++++---
 hack/bin/integration-setup-federation.sh | 3 ++-
 hack/bin/integration-setup.sh            | 3 ++-
 hack/bin/integration-test.sh             | 2 +-
 4 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/Makefile b/Makefile
index 28ba398972..2d9115380a 100644
--- a/Makefile
+++ b/Makefile
@@ -20,6 +20,7 @@ elasticsearch-ephemeral minio-external cassandra-external						\
 nginx-ingress-controller nginx-ingress-services reaper sftd restund coturn		\
 inbucket k8ssandra-test-cluster
 KIND_CLUSTER_NAME     := wire-server
+HELM_PARALLELISM      ?= 1 # 1 for sequential tests; 6 for parallel tests
 
 package ?= all
 EXE_SCHEMA := ./dist/$(package)-schema
@@ -315,15 +316,15 @@ kube-integration:  kube-integration-setup kube-integration-test
 
 .PHONY: kube-integration-setup
 kube-integration-setup: charts-integration
-	export NAMESPACE=$(NAMESPACE); ./hack/bin/integration-setup-federation.sh
+	export NAMESPACE=$(NAMESPACE); export HELM_PARALLELISM=$(HELM_PARALLELISM); ./hack/bin/integration-setup-federation.sh
 
 .PHONY: kube-integration-test
 kube-integration-test:
-	export NAMESPACE=$(NAMESPACE); ./hack/bin/integration-test.sh
+	export NAMESPACE=$(NAMESPACE); export HELM_PARALLELISM=$(HELM_PARALLELISM); ./hack/bin/integration-test.sh
 
 .PHONY: kube-integration-teardown
 kube-integration-teardown:
-	export NAMESPACE=$(NAMESPACE); ./hack/bin/integration-teardown-federation.sh
+	export NAMESPACE=$(NAMESPACE); export HELM_PARALLELISM=$(HELM_PARALLELISM); ./hack/bin/integration-teardown-federation.sh
 
 .PHONY: kube-integration-e2e-telepresence
 kube-integration-e2e-telepresence:
diff --git a/hack/bin/integration-setup-federation.sh b/hack/bin/integration-setup-federation.sh
index 39198ca63b..f569928239 100755
--- a/hack/bin/integration-setup-federation.sh
+++ b/hack/bin/integration-setup-federation.sh
@@ -7,6 +7,7 @@ TOP_LEVEL="$DIR/../.."
 export NAMESPACE=${NAMESPACE:-test-integration}
 HELMFILE_ENV=${HELMFILE_ENV:-default}
 CHARTS_DIR="${TOP_LEVEL}/.local/charts"
+HELM_PARALLELISM=${HELM_PARALLELISM:-1}
 
 . "$DIR/helm_overrides.sh"
 ${DIR}/integration-cleanup.sh
@@ -21,7 +22,7 @@ ${DIR}/integration-cleanup.sh
 echo "updating recursive dependencies ..."
 charts=(fake-aws databases-ephemeral redis-cluster wire-server nginx-ingress-controller nginx-ingress-services)
 mkdir -p ~/.parallel && touch ~/.parallel/will-cite
-printf '%s\n' "${charts[@]}" | parallel "$DIR/update.sh" "$CHARTS_DIR/{}"
+printf '%s\n' "${charts[@]}" | parallel -P "${HELM_PARALLELISM}" "$DIR/update.sh" "$CHARTS_DIR/{}"
 
 # FUTUREWORK: use helm functions instead, see https://wearezeta.atlassian.net/browse/SQPIT-723
 echo "Generating self-signed certificates..."
diff --git a/hack/bin/integration-setup.sh b/hack/bin/integration-setup.sh
index 91cac6bb5f..59cf0e4f84 100755
--- a/hack/bin/integration-setup.sh
+++ b/hack/bin/integration-setup.sh
@@ -7,6 +7,7 @@ TOP_LEVEL="$DIR/../.."
 export NAMESPACE=${NAMESPACE:-test-integration}
 HELMFILE_ENV=${HELMFILE_ENV:-default}
 CHARTS_DIR="${TOP_LEVEL}/.local/charts"
+HELM_PARALLELISM=${HELM_PARALLELISM:-1}
 
 . "$DIR/helm_overrides.sh"
 
@@ -15,7 +16,7 @@ CHARTS_DIR="${TOP_LEVEL}/.local/charts"
 echo "updating recursive dependencies ..."
 charts=(fake-aws databases-ephemeral redis-cluster wire-server nginx-ingress-controller nginx-ingress-services)
 mkdir -p ~/.parallel && touch ~/.parallel/will-cite
-printf '%s\n' "${charts[@]}" | parallel "$DIR/update.sh" "$CHARTS_DIR/{}"
+printf '%s\n' "${charts[@]}" | parallel -P "${HELM_PARALLELISM}" "$DIR/update.sh" "$CHARTS_DIR/{}"
 
 echo "Generating self-signed certificates..."
 export FEDERATION_DOMAIN_BASE="$NAMESPACE.svc.cluster.local"
diff --git a/hack/bin/integration-test.sh b/hack/bin/integration-test.sh
index 8f24c3f000..a143e47e25 100755
--- a/hack/bin/integration-test.sh
+++ b/hack/bin/integration-test.sh
@@ -6,7 +6,7 @@ NAMESPACE=${NAMESPACE:-test-integration}
 # set to 1 to disable running helm tests in parallel
 HELM_PARALLELISM=${HELM_PARALLELISM:-1}
 
-echo "Running integration tests on wire-server"
+echo "Running integration tests on wire-server with parallelism=${HELM_PARALLELISM} ..."
 
 CHART=wire-server
 tests=(cargohold gundeck federator spar brig galley)

From 1f2db54e26402d655ce9bfa5dbf89632cfc1ab34 Mon Sep 17 00:00:00 2001
From: jschaul <jschaul@users.noreply.github.com>
Date: Wed, 1 Feb 2023 19:29:45 +0100
Subject: [PATCH 17/24] parallel tests again

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 2d9115380a..3c94e50fe4 100644
--- a/Makefile
+++ b/Makefile
@@ -20,7 +20,7 @@ elasticsearch-ephemeral minio-external cassandra-external						\
 nginx-ingress-controller nginx-ingress-services reaper sftd restund coturn		\
 inbucket k8ssandra-test-cluster
 KIND_CLUSTER_NAME     := wire-server
-HELM_PARALLELISM      ?= 1 # 1 for sequential tests; 6 for parallel tests
+HELM_PARALLELISM      ?= 6 # 1 for sequential tests; 6 for parallel tests
 
 package ?= all
 EXE_SCHEMA := ./dist/$(package)-schema

From 6b924d1df29a08e51cd39c9944faa1c2b7715c5e Mon Sep 17 00:00:00 2001
From: jschaul <jschaul@users.noreply.github.com>
Date: Thu, 2 Feb 2023 11:26:38 +0100
Subject: [PATCH 18/24] Add colour reset line

---
 hack/bin/integration-logs-relevant-bits.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/hack/bin/integration-logs-relevant-bits.sh b/hack/bin/integration-logs-relevant-bits.sh
index 4ebc9f6c35..bb6fd5f94b 100755
--- a/hack/bin/integration-logs-relevant-bits.sh
+++ b/hack/bin/integration-logs-relevant-bits.sh
@@ -47,6 +47,7 @@ cleanup() {
 grepper() {
     # print 10 lines before/after for context
     rg "$problem_markers" --color=always -A 10 -B 10
+    echo -e "\033[0m"
 }
 
 cleanup | excludeLogEntries | grepper

From fd033c830175961066978b47a4736bf34754852e Mon Sep 17 00:00:00 2001
From: jschaul <jschaul@users.noreply.github.com>
Date: Thu, 2 Feb 2023 11:28:24 +0100
Subject: [PATCH 19/24] try parallelism of 3

---
 Makefile                     | 2 +-
 hack/bin/integration-test.sh | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index 3c94e50fe4..2ffc0c4634 100644
--- a/Makefile
+++ b/Makefile
@@ -20,7 +20,7 @@ elasticsearch-ephemeral minio-external cassandra-external						\
 nginx-ingress-controller nginx-ingress-services reaper sftd restund coturn		\
 inbucket k8ssandra-test-cluster
 KIND_CLUSTER_NAME     := wire-server
-HELM_PARALLELISM      ?= 6 # 1 for sequential tests; 6 for parallel tests
+HELM_PARALLELISM      ?= 3 # 1 for sequential tests; 6 for all-parallel tests
 
 package ?= all
 EXE_SCHEMA := ./dist/$(package)-schema
diff --git a/hack/bin/integration-test.sh b/hack/bin/integration-test.sh
index a143e47e25..50b560df56 100755
--- a/hack/bin/integration-test.sh
+++ b/hack/bin/integration-test.sh
@@ -9,7 +9,7 @@ HELM_PARALLELISM=${HELM_PARALLELISM:-1}
 echo "Running integration tests on wire-server with parallelism=${HELM_PARALLELISM} ..."
 
 CHART=wire-server
-tests=(cargohold gundeck federator spar brig galley)
+tests=(galley cargohold gundeck federator spar brig)
 
 summary() {
     echo "==============="

From 134ea293bad8179e82238152096e25a4fb2a61e0 Mon Sep 17 00:00:00 2001
From: jschaul <jschaul@users.noreply.github.com>
Date: Thu, 2 Feb 2023 11:33:06 +0100
Subject: [PATCH 20/24] cleanup local files

---
 hack/bin/integration-test.sh | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/hack/bin/integration-test.sh b/hack/bin/integration-test.sh
index 50b560df56..47c5db9c3d 100755
--- a/hack/bin/integration-test.sh
+++ b/hack/bin/integration-test.sh
@@ -5,12 +5,22 @@ DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 NAMESPACE=${NAMESPACE:-test-integration}
 # set to 1 to disable running helm tests in parallel
 HELM_PARALLELISM=${HELM_PARALLELISM:-1}
+CLEANUP_LOCAL_FILES=${CLEANUP_LOCAL_FILES:-1} # set to 0 to keep files
 
 echo "Running integration tests on wire-server with parallelism=${HELM_PARALLELISM} ..."
 
 CHART=wire-server
 tests=(galley cargohold gundeck federator spar brig)
 
+cleanup() {
+    if (( CLEANUP_LOCAL_FILES > 0 )); then
+        for t in "${tests[@]}"; do
+            rm -f "stat-$t"
+            rm -f "logs-$t"
+        done
+    fi
+}
+
 summary() {
     echo "==============="
     echo "=== summary ==="
@@ -73,6 +83,8 @@ if ((exit_code > 0)); then
     summary
 fi
 
+cleanup
+
 if ((exit_code > 0)); then
     echo "Tests failed."
     exit 1

From 5782691322b04904cd38ceef72f10a11969d19e9 Mon Sep 17 00:00:00 2001
From: jschaul <jschaul@users.noreply.github.com>
Date: Thu, 2 Feb 2023 12:24:22 +0100
Subject: [PATCH 21/24] use sequential tests for now

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 2ffc0c4634..986013600f 100644
--- a/Makefile
+++ b/Makefile
@@ -20,7 +20,7 @@ elasticsearch-ephemeral minio-external cassandra-external						\
 nginx-ingress-controller nginx-ingress-services reaper sftd restund coturn		\
 inbucket k8ssandra-test-cluster
 KIND_CLUSTER_NAME     := wire-server
-HELM_PARALLELISM      ?= 3 # 1 for sequential tests; 6 for all-parallel tests
+HELM_PARALLELISM      ?= 1 # 1 for sequential tests; 6 for all-parallel tests
 
 package ?= all
 EXE_SCHEMA := ./dist/$(package)-schema

From f07bb60d83e0bf2afffb0ae907a4a03c7add810d Mon Sep 17 00:00:00 2001
From: jschaul <jschaul@users.noreply.github.com>
Date: Thu, 2 Feb 2023 12:46:45 +0100
Subject: [PATCH 22/24] exclude other log entries like

2023-02-02T10:47:22Z, D, Connection established: datacenter1:rack1:10.233.101.194:9042#<socket: 146>
---
 hack/bin/integration-logs-relevant-bits.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/hack/bin/integration-logs-relevant-bits.sh b/hack/bin/integration-logs-relevant-bits.sh
index bb6fd5f94b..96b836ca90 100755
--- a/hack/bin/integration-logs-relevant-bits.sh
+++ b/hack/bin/integration-logs-relevant-bits.sh
@@ -30,7 +30,8 @@ exit_usage() {
 # often this is just noise like connection to cassandra.
 excludeLogEntries() {
     grep -v '^{".*Info"' |
-        grep -v '^{".*Debug"'
+        grep -v '^{".*Debug"' |
+        grep -v '^20.*, D, .*socket: [0-9]\+>$'
 }
 
 cleanup() {

From 3446c7b7cdf7f445e0ba8e85ff3229b79b3ce192 Mon Sep 17 00:00:00 2001
From: jschaul <jschaul@users.noreply.github.com>
Date: Thu, 2 Feb 2023 13:40:38 +0100
Subject: [PATCH 23/24] Hi CI


From 5bd9c0a16fd09df5e800830221a07a95e0e48090 Mon Sep 17 00:00:00 2001
From: jschaul <jschaul@users.noreply.github.com>
Date: Thu, 2 Feb 2023 17:14:14 +0100
Subject: [PATCH 24/24] adjust changelog

---
 changelog.d/5-internal/parallel-helm-tests | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/changelog.d/5-internal/parallel-helm-tests b/changelog.d/5-internal/parallel-helm-tests
index 38e95a823a..3f4909c3b7 100644
--- a/changelog.d/5-internal/parallel-helm-tests
+++ b/changelog.d/5-internal/parallel-helm-tests
@@ -1 +1 @@
-Run helm tests for different services in parallel
+Add config to allow to run helm tests for different services in parallel; improve integration test output logs