Use k3d instead of kind

.github/workflows/pytest.yml

@@ -79,12 +79,11 @@ jobs:
         username: ${{ secrets.DOCKERHUB_USERNAME }}
         password: ${{ secrets.DOCKERHUB_TOKEN }}
 
-    - uses: engineerd/setup-kind@ecfad61750951586a9ef973db567df1d28671bdc  # v0.6.2
+    - name: Install k3d with asdf
+      uses: asdf-vm/actions/install@1902764435ca0dd2f3388eea723a4f92a4eb8302  # v4
       with:
-        version: "v0.29.0"
-        name: "ess-helm"
-        skipClusterCreation: "true"
-        skipClusterLogsExport: "true"
+        tool_versions: |
+          k3d 5.8.3
 
     - uses: azure/setup-helm@1a275c3b69536ee54be43f2070a358922e12c8d4  # v4.3.1
 
@@ -124,17 +123,23 @@ jobs:
       if: ${{ failure() }}
       shell: bash
       run: |
-        kind export logs --name ess-helm ./ess-helm-logs
-        kind export kubeconfig --name ess-helm
-        ns=$(kubectl --context kind-ess-helm get ns -l app.kubernetes.io/managed-by=pytest  -o jsonpath='{.items[].metadata.name}')
+        mkdir ess-helm-logs
+        k3d kubeconfig merge ess-helm -ds
+        for ns in $(kubectl --context k3d-ess-helm get ns -o custom-columns=NS:.metadata.name --no-headers); do
+          mkdir -p "./ess-helm-logs/$ns"
+          for pod in $(kubectl --context k3d-ess-helm -n "$ns" get pod -o custom-columns=NS:.metadata.name --no-headers --field-selector='status.phase!=Pending'); do
+            kubectl --context k3d-ess-helm -n "$ns" logs --all-containers --prefix --timestamps "$pod" > "./ess-helm-logs/$ns/$pod.logs"
+          done
+        done
+        ess_ns=$(kubectl --context k3d-ess-helm get ns -l app.kubernetes.io/managed-by=pytest -o jsonpath='{.items[].metadata.name}')
         resources=("pods" "deployments" "statefulsets" "services" "configmaps" "ingresses" "persistentvolumes" "persistentvolumeclaims" "endpoints")
         for i in "${resources[@]}"; do
-          kubectl --context kind-ess-helm get "$i" -n "$ns" > "./ess-helm-logs/$i.txt"
+          kubectl --context k3d-ess-helm get "$i" -n "$ess_ns" > "./ess-helm-logs/$i.txt"
           echo "----" >> "./ess-helm-logs/$i.txt"
-          kubectl --context kind-ess-helm get "$i" -o yaml -n "$ns" >> "./ess-helm-logs/$i.txt"
+          kubectl --context k3d-ess-helm get "$i" -o yaml -n "$ess_ns" >> "./ess-helm-logs/$i.txt"
         done
-        kubectl --context kind-ess-helm get events --sort-by=.metadata.creationTimestamp -n "$ns" > ./ess-helm-logs/events.txt
-        kind delete cluster --name ess-helm
+        kubectl --context k3d-ess-helm get events --sort-by=.metadata.creationTimestamp -n "$ess_ns" > ./ess-helm-logs/events.txt
+        k3d cluster delete ess-helm
 
     - name: Upload logs
       if: ${{ failure() }}

charts/matrix-stack/ci/fragments/well-known-pytest-base-extras.yaml

@@ -3,9 +3,6 @@
 #
 # SPDX-License-Identifier: AGPL-3.0-only
 
-ingress:
-  controllerType: ingress-nginx
-
 wellKnownDelegation:
   ingress:
     tlsSecret: "{{ $.Release.Name }}-well-known-web-tls"

charts/matrix-stack/ci/pytest-matrix-rtc-synapse-wellknown-values.yaml

@@ -16,8 +16,6 @@ haproxy:
   podSecurityContext:
     runAsGroup: 0
   replicas: 2
-ingress:
-  controllerType: ingress-nginx
 initSecrets:
   annotations:
     has-no-service-monitor: "true"

charts/matrix-stack/ci/pytest-well-known-values.yaml

@@ -18,8 +18,6 @@ global:
 haproxy:
   podSecurityContext:
     runAsGroup: 0
-ingress:
-  controllerType: ingress-nginx
 matrixAuthenticationService:
   enabled: false
 matrixRTC:

charts/matrix-stack/ci/test-cluster-mixin.yaml

@@ -10,9 +10,6 @@
 certManager:
   clusterIssuer: ess-selfsigned
 
-ingress:
-  controllerType: ingress-nginx
-
 matrixRTC:
   # Because the authoriser service won't trust certificates issued by the above self-signed CA
   extraEnv:
@@ -24,4 +21,4 @@ matrixRTC:
     - ess.localhost
     - mrtc.ess.localhost
     - synapse.ess.localhost
-    ip: '{{ ( (lookup "v1" "Service" "ingress-nginx" "ingress-nginx-controller") | default (dict "spec" (dict "clusterIP" "127.0.0.1")) ).spec.clusterIP }}'
+    ip: '{{ ( (lookup "v1" "Service" "kube-system" "traefik") | default (dict "spec" (dict "clusterIP" "127.0.0.1")) ).spec.clusterIP }}'

newsfragments/871.internal.md

@@ -0,0 +1 @@
+CI: switch from kind to k3d for integration tests.

pyproject.toml

@@ -41,6 +41,7 @@ types-pyyaml = "^6.0.12.20250915"
 semver = "^3.0.4"
 prometheus-client = "^0.23.1"
 yamllint = "^1.37.1"
+httpx-retries = "^0.4.5"
 
 [build-system]
 requires = ["poetry-core>=2.1.0"]

scripts/destroy_test_cluster.sh

@@ -7,17 +7,10 @@
 
 set -e
 
-kind_cluster_name="ess-helm"
+k3d_cluster_name="ess-helm"
 
-if kind get clusters 2> /dev/null| grep "$kind_cluster_name"; then
-  kind delete cluster --name $kind_cluster_name
+if k3d cluster list 2> /dev/null | grep "$k3d_cluster_name"; then
+  k3d cluster delete $k3d_cluster_name
 else
-  echo "Kind cluster ${kind_cluster_name} already destoryed"
-fi
-
-if docker ps -a | grep "${kind_cluster_name}-registry"; then
-  docker stop "${kind_cluster_name}-registry" || true
-  docker rm "${kind_cluster_name}-registry" || true
-else
-  echo "Kind cluster's local registry already destroyed"
+  echo "k3d cluster ${k3d_cluster_name} already destoryed"
 fi

scripts/setup_test_cluster.sh

@@ -7,52 +7,34 @@
 
 set -e
 
-kind_cluster_name="ess-helm"
-kind_context_name="kind-$kind_cluster_name"
+k3d_cluster_name="ess-helm"
+k3d_context_name="k3d-$k3d_cluster_name"
 # Space separated list of namespaces to use
 ess_namespaces=${ESS_NAMESPACES:-ess}
 
 root_folder="$(git rev-parse --show-toplevel)"
 ca_folder="$root_folder/.ca"
 mkdir -p "$ca_folder"
 
-if docker ps -a | grep "${kind_cluster_name}-registry"; then
-  docker stop "${kind_cluster_name}-registry" || true
-  docker rm "${kind_cluster_name}-registry" || true
-fi
-if kind get clusters 2>/dev/null | grep "$kind_cluster_name"; then
-  echo "Cluster '$kind_cluster_name' is already provisioned by Kind"
+if k3d cluster list 2>/dev/null | grep "$k3d_cluster_name"; then
+  echo "Cluster '$k3d_cluster_name' is already provisioned by k3d"
 else
-  echo "Creating new Kind cluster '$kind_cluster_name'"
-  (cd "$root_folder/tests/integration/fixtures/files/clusters"; kind create cluster --name "$kind_cluster_name" --config "kind.yml")
+  echo "Creating new k3d cluster '$k3d_cluster_name'"
+  k3d cluster create "$k3d_cluster_name" --config "tests/integration/fixtures/files/clusters/k3d.yml"
 fi
 
-network=$(docker inspect $kind_cluster_name-control-plane | jq '.[0].NetworkSettings.Networks | keys | .[0]' -r)
-docker run \
-    -d --restart=always -p "127.0.0.1:5000:5000" --network "$network" --network-alias "registry" --name "${kind_cluster_name}-registry" \
-    registry:2
-
-helm --kube-context $kind_context_name upgrade -i ingress-nginx --repo https://kubernetes.github.io/ingress-nginx ingress-nginx \
-  --namespace ingress-nginx \
-  --create-namespace \
-  -f "$root_folder/tests/integration/fixtures/files/charts/ingress-nginx.yml"
-
-helm --kube-context $kind_context_name upgrade -i metrics-server --repo https://kubernetes-sigs.github.io/metrics-server metrics-server \
-  --namespace kube-system \
-  -f "$root_folder/tests/integration/fixtures/files/charts/metrics-server.yml"
-
-helm --kube-context $kind_context_name upgrade -i prometheus-operator-crds --repo https://prometheus-community.github.io/helm-charts prometheus-operator-crds \
+helm --kube-context $k3d_context_name upgrade -i prometheus-operator-crds --repo https://prometheus-community.github.io/helm-charts prometheus-operator-crds \
   --namespace prometheus-operator \
   --create-namespace
 
-helm --kube-context $kind_context_name upgrade -i cert-manager --repo https://charts.jetstack.io cert-manager \
+helm --kube-context $k3d_context_name upgrade -i cert-manager --repo https://charts.jetstack.io cert-manager \
   --namespace cert-manager \
   --create-namespace \
   -f "$root_folder/tests/integration/fixtures/files/charts/cert-manager.yml"
 
 # Create a new CA certificate
 if [[ ! -f "$ca_folder"/ca.crt || ! -f "$ca_folder"/ca.pem ]]; then
-  cat <<EOF | kubectl --context $kind_context_name apply -f -
+  cat <<EOF | kubectl --context $k3d_context_name apply -f -
 ---
 apiVersion: cert-manager.io/v1
 kind: ClusterIssuer
@@ -80,19 +62,19 @@ spec:
     group: cert-manager.io
 ---
 EOF
-  kubectl --context $kind_context_name -n cert-manager wait --for condition=Ready Certificate/ess-ca
+  kubectl --context $k3d_context_name -n cert-manager wait --for condition=Ready Certificate/ess-ca
 else
-  kubectl --context $kind_context_name delete ClusterIssuer ess-ca 2>/dev/null || true
-  kubectl --context $kind_context_name -n cert-manager delete Certificate ess-ca 2>/dev/null || true
-  kubectl --context $kind_context_name -n cert-manager delete Secret ess-ca 2>/dev/null || true
-  kubectl --context $kind_context_name -n cert-manager create secret generic ess-ca \
+  kubectl --context $k3d_context_name delete ClusterIssuer ess-ca 2>/dev/null || true
+  kubectl --context $k3d_context_name -n cert-manager delete Certificate ess-ca 2>/dev/null || true
+  kubectl --context $k3d_context_name -n cert-manager delete Secret ess-ca 2>/dev/null || true
+  kubectl --context $k3d_context_name -n cert-manager create secret generic ess-ca \
     --type=kubernetes.io/tls \
     --from-file=tls.crt="$ca_folder"/ca.crt \
     --from-file=tls.key="$ca_folder"/ca.pem \
     --from-file=ca.crt="$ca_folder"/ca.crt
 fi
 
-cat <<EOF | kubectl --context $kind_context_name apply -f -
+cat <<EOF | kubectl --context $k3d_context_name apply -f -
 apiVersion: cert-manager.io/v1
 kind: ClusterIssuer
 metadata:
@@ -103,15 +85,15 @@ spec:
 EOF
 
 if [[ ! -f "$ca_folder"/ca.crt || ! -f "$ca_folder"/ca.pem ]]; then
-  kubectl --context $kind_context_name -n cert-manager get secret ess-ca -o jsonpath="{.data['ca\.crt']}" | base64 -d > "$ca_folder"/ca.crt
-  kubectl --context $kind_context_name -n cert-manager get secret ess-ca -o jsonpath="{.data['tls\.key']}" | base64 -d > "$ca_folder"/ca.pem
+  kubectl --context $k3d_context_name -n cert-manager get secret ess-ca -o jsonpath="{.data['ca\.crt']}" | base64 -d > "$ca_folder"/ca.crt
+  kubectl --context $k3d_context_name -n cert-manager get secret ess-ca -o jsonpath="{.data['tls\.key']}" | base64 -d > "$ca_folder"/ca.pem
 fi
 
 for namespace in $ess_namespaces; do
   echo "Constructing ESS dependencies in $namespace"
-  server_version=$(kubectl --context $kind_context_name version | grep Server | sed 's/.*v/v/' | awk -F. '{print $1"."$2}')
+  server_version=$(kubectl --context $k3d_context_name version | grep Server | sed 's/.*v/v/' | awk -F. '{print $1"."$2}')
   # We don't turn on enforce here as people may be experimenting but we do turn on warn so people see the warnings when helm install/upgrade
-  cat <<EOF | kubectl --context $kind_context_name apply -f -
+  cat <<EOF | kubectl --context $k3d_context_name apply -f -
 apiVersion: v1
 kind: Namespace
 metadata:

tests/integration/fixtures/__init__.py

@@ -4,7 +4,7 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 from .ca import delegated_ca, root_ca, ssl_context
-from .cluster import cluster, ess_namespace, helm_client, ingress, kube_client, prometheus_operator_crds, registry
+from .cluster import cluster, ess_namespace, helm_client, ingress, kube_client, prometheus_operator_crds
 from .data import ESSData, generated_data
 from .helm import helm_prerequisites, ingress_ready, matrix_stack, secrets_generated
 from .matrix_tools import build_matrix_tools, loaded_matrix_tools
@@ -25,7 +25,6 @@
     "loaded_matrix_tools",
     "matrix_stack",
     "prometheus_operator_crds",
-    "registry",
     "root_ca",
     "secrets_generated",
     "ssl_context",