Use helmfile's parallelism to speed up integration test setup time

Makefile

@@ -311,7 +311,7 @@ release-chart-%:
 .PHONY: guard-tag
 guard-tag:
 	@if [ "${DOCKER_TAG}" = "${USER}" ]; then \
-	      echo "Environment variable DOCKER_TAG not set to non-default value. Re-run with DOCKER_TAG=<something>. Try using 'make latest-brig-tag' for latest develop docker image tag";\
+	      echo "Environment variable DOCKER_TAG not set to non-default value. Re-run with DOCKER_TAG=<something>. Try using 'make latest-tag' for latest develop docker image tag";\
 	    exit 1; \
 	fi
 

changelog.d/5-internal/helmfile

@@ -0,0 +1 @@
+Internal CI tooling improvement: decrease integration setup time by using helmfile.

charts/fake-aws-s3/values.yaml

@@ -7,6 +7,8 @@ minio:
     enabled: false
   environment:
     MINIO_BROWSER: "off"
+  defaultBucket:
+    name: dummy-bucket
   buckets:
     - name: dummy-bucket
       purge: true

docs/developer/dependencies.md

@@ -225,4 +225,4 @@ See `make buildah-docker` for an entry point here.
 
 ## Helm chart development, integration tests in kubernetes
 
-You need `kubectl`, `helm`, and a valid kubernetes context. Refer to https://docs.wire.com for details.
+You need `kubectl`, `helm`, `helmfile`, and a valid kubernetes context. Refer to https://docs.wire.com for details.

hack/bin/helm_overrides.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+# Helm (v3) writes into XDG folders only these days. They don't honor HELM_ vars
+# anymore.
+# Derive a helm-specific folder inside the wire-server/.local to avoid polluting
+# ~.
+
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+TOP_LEVEL="$DIR/../.."
+LOCAL_HELM_FOLDER="$TOP_LEVEL/.local/helm"
+
+[[ -e $LOCAL_HELM_FOLDER ]] || mkdir -p "$LOCAL_HELM_FOLDER"
+export XDG_CACHE_HOME=${LOCAL_HELM_FOLDER}/cache
+export XDG_CONFIG_HOME=${LOCAL_HELM_FOLDER}/config
+export XDG_DATA_HOME=${LOCAL_HELM_FOLDER}/data

hack/bin/integration-setup-federation.sh

@@ -1,19 +1,56 @@
 #!/usr/bin/env bash
 
-USAGE="Usage: $0"
+set -euo pipefail
 
-set -e
-
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 TOP_LEVEL="$DIR/../.."
-
 export NAMESPACE=${NAMESPACE:-test-integration}
+CHARTS_DIR="${TOP_LEVEL}/.local/charts"
+
+. "$DIR/helm_overrides.sh"
+${DIR}/integration-cleanup.sh
+
+# FUTUREWORK explore: have helmfile do the interpolation (and skip the "make charts" step) https://wearezeta.atlassian.net/browse/SQPIT-722
+#
+# FUTUREWORK: get rid of wrapper charts, use helmfile for pinning. Then we may not need the recursive update hack anymore: https://wearezeta.atlassian.net/browse/SQPIT-721
+#
+# Sadly, even with helmfile, we still need to use use this recursive update
+# script beforehand on all relevant charts to download the nested dependencies
+# (e.g. cassandra from underneath databases-ephemeral)
+echo "updating recursive dependencies ..."
+charts=(fake-aws databases-ephemeral wire-server nginx-ingress-controller nginx-ingress-services)
+for chart in "${charts[@]}"; do
+    "$DIR/update.sh" "$CHARTS_DIR/$chart"
+done
+
+# FUTUREWORK: use helm functions instead, see https://wearezeta.atlassian.net/browse/SQPIT-723
+echo "Generating self-signed certificates..."
+
+export NAMESPACE_1="$NAMESPACE"
+export FEDERATION_DOMAIN_BASE="$NAMESPACE_1.svc.cluster.local"
+export FEDERATION_DOMAIN_1="federation-test-helper.$FEDERATION_DOMAIN_BASE"
+"$DIR/selfsigned-kubernetes.sh" namespace1
+
+export NAMESPACE_2="$NAMESPACE-fed2"
+export FEDERATION_DOMAIN_BASE="$NAMESPACE_2.svc.cluster.local"
+export FEDERATION_DOMAIN_2="federation-test-helper.$FEDERATION_DOMAIN_BASE"
+"$DIR/selfsigned-kubernetes.sh" namespace2
+
+echo "Installing charts..."
+
+helmfile --file ${TOP_LEVEL}/hack/helmfile.yaml sync
 
-$DIR/integration-setup.sh
+# wait for fakeSNS to create resources. TODO, cleaner: make initiate-fake-aws-sns a post hook. See cassandra-migrations chart for an example.
+resourcesReady() {
+    SNS_POD=$(kubectl -n "${NAMESPACE_1}" get pods | grep fake-aws-sns | grep Running | awk '{print $1}')
+    kubectl -n "${NAMESPACE_1}" logs "$SNS_POD" -c initiate-fake-aws-sns | grep created
 
-# The suffix '-fed2' must be kept in sync with configuration inside
-# charts/brig/templates/tests/configmap.yaml and
-# hack/bin/integration-teardown-federation.sh
-export NAMESPACE=${NAMESPACE}-fed2
+    SNS_POD=$(kubectl -n "${NAMESPACE_2}" get pods | grep fake-aws-sns | grep Running | awk '{print $1}')
+    kubectl -n "${NAMESPACE_2}" logs "$SNS_POD" -c initiate-fake-aws-sns | grep created
+}
+until resourcesReady; do
+    echo 'waiting for SNS resources'
+    sleep 1
+done
 
-$DIR/integration-setup.sh
+echo "done"

hack/bin/integration-setup.sh

@@ -1,84 +1,30 @@
 #!/usr/bin/env bash
 
-USAGE="Usage: $0"
-
-set -e
+set -euo pipefail
 
 DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 TOP_LEVEL="$DIR/../.."
+export NAMESPACE=${NAMESPACE:-test-integration}
 CHARTS_DIR="${TOP_LEVEL}/.local/charts"
 
-NAMESPACE=${NAMESPACE:-test-integration}
-ENABLE_KIND_VALUES=${ENABLE_KIND_VALUES:-0}
-
-kubectl create namespace "${NAMESPACE}" >/dev/null 2>&1 || true
+. "$DIR/helm_overrides.sh"
 
-${DIR}/integration-cleanup.sh
-
-charts=(fake-aws databases-ephemeral wire-server nginx-ingress-controller nginx-ingress-services)
+"${DIR}/integration-cleanup.sh"
 
 echo "updating recursive dependencies ..."
+charts=(fake-aws databases-ephemeral wire-server nginx-ingress-controller nginx-ingress-services)
 for chart in "${charts[@]}"; do
     "$DIR/update.sh" "$CHARTS_DIR/$chart"
 done
 
-echo "Installing charts..."
-
-function printLogs() {
-    echo "---- a command failed, attempting to print useful debug information..."
-    echo "-------------------------------"
-    echo "-------------------------------"
-    echo "-------------------------------"
-    echo ""
-    kubectl -n ${NAMESPACE} get pods
-    kubectl -n ${NAMESPACE} get pods | grep -v Running | grep -v Pending | grep -v Completed | grep -v STATUS | grep -v ContainerCreating | awk '{print $1}' | xargs -n 1 -I{} bash -c "printf '\n\n----LOGS FROM {}:\n'; kubectl -n ${NAMESPACE} logs --tail=30 {}" || true
-    kubectl -n ${NAMESPACE} get pods | grep Pending | awk '{print $1}' | xargs -n 1 -I{} bash -c "printf '\n\n----DESCRIBE 'pending' {}:\n'; kubectl -n ${NAMESPACE} describe pod {}" || true
-}
-
-trap printLogs ERR
-
+echo "Generating self-signed certificates..."
 export FEDERATION_DOMAIN_BASE="$NAMESPACE.svc.cluster.local"
-FEDERATION_DOMAIN="federation-test-helper.$FEDERATION_DOMAIN_BASE"
-"$DIR/selfsigned-kubernetes.sh"
-
-for chart in "${charts[@]}"; do
-    kubectl -n ${NAMESPACE} get pods
-    valuesfile="${DIR}/../helm_vars/${chart}/values.yaml"
-    kindValuesfile="${DIR}/../helm_vars/${chart}/kind-values.yaml"
-    certificatesValuesfile="${DIR}/../helm_vars/${chart}/certificates.yaml"
-
-    declare -a options=()
-
-    if [ -f "$valuesfile" ]; then
-        options+=(-f "$valuesfile")
-    fi
+export FEDERATION_DOMAIN="federation-test-helper.$FEDERATION_DOMAIN_BASE"
+"$DIR/selfsigned-kubernetes.sh" namespace1
 
-    if [ -f "$certificatesValuesfile" ]; then
-        options+=(-f "$certificatesValuesfile")
-    fi
-
-    if [[ "$chart" == "nginx-ingress-services" ]]; then
-        # Federation domain is also the SRV record created by the
-        # federation-test-helper service. Maybe we can find a way to make these
-        # differ, so we don't make any silly assumptions in the code.
-        options+=("--set" "config.dns.federator=$FEDERATION_DOMAIN")
-    fi
-
-    if [[ "$ENABLE_KIND_VALUES" == "1" ]] && [[ -f "$kindValuesfile" ]]; then
-        options+=(-f "$kindValuesfile")
-    fi
+echo "Installing charts..."
 
-    # default is 5m but may not be enough on a fresh install including cassandra migrations
-    TIMEOUT=10m
-    set -x
-    helm upgrade --install --namespace "${NAMESPACE}" "${NAMESPACE}-${chart}" "${CHARTS_DIR}/${chart}" \
-        ${options[*]} \
-        --set brig.config.optSettings.setFederationDomain="$FEDERATION_DOMAIN" \
-        --set galley.config.settings.federationDomain="$FEDERATION_DOMAIN" \
-        --wait \
-        --timeout "$TIMEOUT"
-    set +x
-done
+helmfile --file "${TOP_LEVEL}/hack/helmfile-single.yaml" sync
 
 # wait for fakeSNS to create resources. TODO, cleaner: make initiate-fake-aws-sns a post hook. See cassandra-migrations chart for an example.
 resourcesReady() {
@@ -90,6 +36,6 @@ until resourcesReady; do
     sleep 1
 done
 
-kubectl -n ${NAMESPACE} get pods
+kubectl -n "${NAMESPACE}" get pods
 
 echo "done"

hack/bin/integration-teardown-federation.sh

@@ -1,16 +1,16 @@
 #!/usr/bin/env bash
 
-set -e
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+TOP_LEVEL="$DIR/../.."
 
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+set -ex
 
-export NAMESPACE=${NAMESPACE:-test-integration}
+NAMESPACE=${NAMESPACE:-test-integration}
+export NAMESPACE_1="$NAMESPACE"
+export NAMESPACE_2="$NAMESPACE-fed2"
+# these don't matter for destruction but have to be set.
+export FEDERATION_DOMAIN_1="."
+export FEDERATION_DOMAIN_2="."
 
-$DIR/integration-teardown.sh
-
-# The suffix '-fed2' must be kept in sync with configuration inside
-# charts/brig/templates/tests/configmap.yaml and
-# hack/bin/integration-setup-federation.sh
-export NAMESPACE=${NAMESPACE}-fed2
-
-$DIR/integration-teardown.sh
+. "$DIR/helm_overrides.sh"
+helmfile --file "${TOP_LEVEL}/hack/helmfile.yaml" destroy

hack/bin/integration-teardown.sh

@@ -1,15 +1,13 @@
 #!/usr/bin/env bash
 
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/.."
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+TOP_LEVEL="$DIR/../.."
 
 NAMESPACE=${NAMESPACE:-test-integration}
+# doesn't matter for destruction but needs to be set
+export FEDERATION_DOMAIN="."
 
 set -ex
 
-echo "NAMESPACE = $NAMESPACE"
-
-helm ls --all --namespace ${NAMESPACE} | grep -v NAME | awk '{print $1}' | xargs -n 1 helm -n "$NAMESPACE" delete
-
-sleep 10
-
-kubectl delete namespace ${NAMESPACE}
+. "$DIR/helm_overrides.sh"
+helmfile --file "${TOP_LEVEL}/hack/helmfile-single.yaml" destroy

hack/bin/selfsigned-kubernetes.sh

@@ -5,16 +5,17 @@
 # These certificates are only meant for integration tests.
 # (The CA certificates are assumed to be re-used across the domains A and B for end2end integration tests.)
 
-set -ex
+set -e
+SUFFIX=${1:?"need suffix argument"}
 TEMP=${TEMP:-/tmp}
 CSR="$TEMP/csr.json"
 OUTPUTNAME_CA="integration-ca"
 OUTPUTNAME_LEAF_CERT="integration-leaf"
 OUTPUTNAME_CLIENT_CERT="integration-client"
 DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 TOP_LEVEL="$DIR/../.."
-OUTPUT_CONFIG_FEDERATOR="$TOP_LEVEL/hack/helm_vars/wire-server/certificates.yaml"
-OUTPUT_CONFIG_INGRESS="$TOP_LEVEL/hack/helm_vars/nginx-ingress-services/certificates.yaml"
+OUTPUT_CONFIG_FEDERATOR="$TOP_LEVEL/hack/helm_vars/wire-server/certificates-$SUFFIX.yaml"
+OUTPUT_CONFIG_INGRESS="$TOP_LEVEL/hack/helm_vars/nginx-ingress-services/certificates-$SUFFIX.yaml"
 
 command -v cfssl >/dev/null 2>&1 || {
     echo >&2 "cfssl is not installed, aborting. See https://github.com/cloudflare/cfssl"
@@ -70,7 +71,7 @@ cfssl gencert -ca "$OUTPUTNAME_CA.pem" -ca-key "$OUTPUTNAME_CA-key.pem" -hostnam
     sed -e 's/^/    /' $OUTPUTNAME_LEAF_CERT-key.pem
     echo "  tlsClientCA: |"
     sed -e 's/^/    /' $OUTPUTNAME_CA.pem
-} | tee "$OUTPUT_CONFIG_INGRESS"
+} >"$OUTPUT_CONFIG_INGRESS"
 
 # the following yaml override file is needed as an override to
 # the wire-server (federator) helm chart
@@ -85,7 +86,7 @@ cfssl gencert -ca "$OUTPUTNAME_CA.pem" -ca-key "$OUTPUTNAME_CA-key.pem" -hostnam
     sed -e 's/^/    /' $OUTPUTNAME_CLIENT_CERT.pem
     echo "  clientPrivateKeyContents: |"
     sed -e 's/^/    /' $OUTPUTNAME_CLIENT_CERT-key.pem
-} | tee "$OUTPUT_CONFIG_FEDERATOR"
+} >"$OUTPUT_CONFIG_FEDERATOR"
 
 # cleanup unneeded files
 rm "$OUTPUTNAME_LEAF_CERT.csr"

hack/helm_vars/.gitignore

@@ -1 +1,3 @@
 certificates.yaml
+certificates-namespace1.yaml
+certificates-namespace2.yaml

hack/helmfile-single.yaml

@@ -0,0 +1,65 @@
+# This helmfile is similar to the 'helmfile.yaml', but only spawns up components for a single backend.
+# In some situations (when not testing anything federation specific), use of a single backend is sufficient.
+#
+# The 'make kube-integration-setup-sans-federation' target uses this helmfile.
+
+helmDefaults:
+  wait: true
+  timeout: 600
+  devel: true
+
+environments:
+  default:
+    values:
+      - namespace: {{ requiredEnv "NAMESPACE" }}
+      - federationDomain: {{ requiredEnv "FEDERATION_DOMAIN" }}
+
+repositories:
+  - name: stable
+    url: 'https://charts.helm.sh/stable'
+
+releases:
+  - name: '{{ .Values.namespace }}-fake-aws'
+    namespace: '{{ .Values.namespace }}'
+    chart: '../.local/charts/fake-aws'
+    values:
+      - './helm_vars/fake-aws/values.yaml'
+
+  - name: '{{ .Values.namespace }}-databases-ephemeral'
+    namespace: '{{ .Values.namespace }}'
+    chart: '../.local/charts/databases-ephemeral'
+
+  - name: '{{ .Values.namespace }}-nginx-ingress-controller'
+    namespace: '{{ .Values.namespace }}'
+    chart: '../.local/charts/nginx-ingress-controller'
+    values:
+      - './helm_vars/nginx-ingress-controller/values.yaml'
+
+  - name: '{{ .Values.namespace }}-nginx-ingress-services'
+    namespace: '{{ .Values.namespace }}'
+    chart: '../.local/charts/nginx-ingress-services'
+    values:
+      - './helm_vars/nginx-ingress-services/values.yaml'
+      - './helm_vars/nginx-ingress-services/certificates-namespace1.yaml'
+    set:
+      # Federation domain is also the SRV record created by the
+      # federation-test-helper service. Maybe we can find a way to make these
+      # differ, so we don't make any silly assumptions in the code.
+      - name: config.dns.federator
+        value: {{ .Values.federationDomain }}
+
+  # Note that wire-server depends on databases-ephemeral being up; and in some
+  # cases on nginx-ingress also being up. If installing helm charts in a
+  # parallel way, it's expected to see some wire-server pods (namely the
+  # cassandra-migration one) fail and get restarted a few times)
+  - name: '{{ .Values.namespace }}-wire-server'
+    namespace: '{{ .Values.namespace }}'
+    chart: '../.local/charts/wire-server'
+    values:
+      - './helm_vars/wire-server/values.yaml'
+      - './helm_vars/wire-server/certificates-namespace1.yaml'
+    set:
+      - name: brig.config.optSettings.setFederationDomain
+        value: {{ .Values.federationDomain }}
+      - name: galley.config.settings.federationDomain
+        value: {{ .Values.federationDomain }}