diff --git a/changelog.d/5-internal/helm-test b/changelog.d/5-internal/helm-test
new file mode 100644
index 0000000000..14bfeb22a3
--- /dev/null
+++ b/changelog.d/5-internal/helm-test
@@ -0,0 +1 @@
+charts: Mark all test resources to be only created while running tests
\ No newline at end of file
diff --git a/charts/brig/templates/tests/brig-integration.yaml b/charts/brig/templates/tests/brig-integration.yaml
index 0687604fb2..81020f9d00 100644
--- a/charts/brig/templates/tests/brig-integration.yaml
+++ b/charts/brig/templates/tests/brig-integration.yaml
@@ -2,6 +2,8 @@ apiVersion: v1
 kind: Service
 metadata:
   name: "brig-integration"
+  annotations:
+    "helm.sh/hook": test
   labels:
     app: brig-integration
     chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
@@ -19,7 +21,7 @@ kind: Pod
 metadata:
   name: "{{ .Release.Name }}-brig-integration"
   annotations:
-    "helm.sh/hook": test-success
+    "helm.sh/hook": test
   labels:
     app: brig-integration
     release: {{ .Release.Name }}
diff --git a/charts/brig/templates/tests/configmap.yaml b/charts/brig/templates/tests/configmap.yaml
index 01721ebf18..4d409eb59f 100644
--- a/charts/brig/templates/tests/configmap.yaml
+++ b/charts/brig/templates/tests/configmap.yaml
@@ -2,6 +2,8 @@ apiVersion: v1
 kind: ConfigMap
 metadata:
   name: "brig-integration"
+  annotations:
+    "helm.sh/hook": test
 data:
   integration.yaml: |
     brig:
diff --git a/charts/brig/templates/tests/nginz-service.yaml b/charts/brig/templates/tests/nginz-service.yaml
index c31128667c..ec1b22284b 100644
--- a/charts/brig/templates/tests/nginz-service.yaml
+++ b/charts/brig/templates/tests/nginz-service.yaml
@@ -5,6 +5,8 @@ apiVersion: v1
 kind: Service
 metadata:
   name: nginz-integration-http
+  annotations:
+    "helm.sh/hook": test
 spec:
   type: ClusterIP
   ports:
diff --git a/charts/brig/templates/tests/secret.yaml b/charts/brig/templates/tests/secret.yaml
index bfe877caf7..bda96d4f3f 100644
--- a/charts/brig/templates/tests/secret.yaml
+++ b/charts/brig/templates/tests/secret.yaml
@@ -2,6 +2,8 @@ apiVersion: v1
 kind: ConfigMap
 metadata:
   name: brig-integration-secrets
+  annotations:
+    "helm.sh/hook": test
 data:
   # These "secrets" are only used in tests and are therefore safe to be stored unencrypted
   provider-privatekey.pem: |
diff --git a/charts/cargohold/templates/tests/cargohold-integration.yaml b/charts/cargohold/templates/tests/cargohold-integration.yaml
index 6decd33e47..722d138637 100644
--- a/charts/cargohold/templates/tests/cargohold-integration.yaml
+++ b/charts/cargohold/templates/tests/cargohold-integration.yaml
@@ -3,7 +3,7 @@ kind: Pod
 metadata:
   name: "{{ .Release.Name }}-cargohold-integration"
   annotations:
-    "helm.sh/hook": test-success
+    "helm.sh/hook": test
 spec:
   volumes:
     - name: "cargohold-integration"
diff --git a/charts/cargohold/templates/tests/configmap.yaml b/charts/cargohold/templates/tests/configmap.yaml
index bb0ff67c8f..1a52056adb 100644
--- a/charts/cargohold/templates/tests/configmap.yaml
+++ b/charts/cargohold/templates/tests/configmap.yaml
@@ -2,6 +2,8 @@ apiVersion: v1
 kind: ConfigMap
 metadata:
   name: "cargohold-integration"
+  annotations:
+    "helm.sh/hook": test
 data:
   integration.yaml: |
     cargohold:
diff --git a/charts/federator/templates/tests/configmap.yaml b/charts/federator/templates/tests/configmap.yaml
index 910411fe5d..b845f3d401 100644
--- a/charts/federator/templates/tests/configmap.yaml
+++ b/charts/federator/templates/tests/configmap.yaml
@@ -2,6 +2,8 @@ apiVersion: v1
 kind: ConfigMap
 metadata:
   name: "federator-integration"
+  annotations:
+    "helm.sh/hook": test
 data:
   integration.yaml: |
     federatorInternal:
diff --git a/charts/federator/templates/tests/federator-integration.yaml b/charts/federator/templates/tests/federator-integration.yaml
index 32e6eef09e..3341cf4173 100644
--- a/charts/federator/templates/tests/federator-integration.yaml
+++ b/charts/federator/templates/tests/federator-integration.yaml
@@ -3,7 +3,7 @@ kind: Pod
 metadata:
   name: "{{ .Release.Name }}-federator-integration"
   annotations:
-    "helm.sh/hook": test-success
+    "helm.sh/hook": test
 spec:
   volumes:
     - name: "federator-integration"
diff --git a/charts/galley/templates/tests/configmap.yaml b/charts/galley/templates/tests/configmap.yaml
index 10d65d5b65..20036f2f15 100644
--- a/charts/galley/templates/tests/configmap.yaml
+++ b/charts/galley/templates/tests/configmap.yaml
@@ -2,6 +2,8 @@ apiVersion: v1
 kind: ConfigMap
 metadata:
   name: "galley-integration"
+  annotations:
+    "helm.sh/hook": test
 data:
   integration.yaml: |
     galley:
diff --git a/charts/galley/templates/tests/galley-integration.yaml b/charts/galley/templates/tests/galley-integration.yaml
index 7d6efa6e4b..d7b64bf5ba 100644
--- a/charts/galley/templates/tests/galley-integration.yaml
+++ b/charts/galley/templates/tests/galley-integration.yaml
@@ -2,6 +2,8 @@ apiVersion: v1
 kind: Service
 metadata:
   name: "galley-integration"
+  annotations:
+    "helm.sh/hook": test
   labels:
     app: galley-integration
     chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
@@ -19,7 +21,7 @@ kind: Pod
 metadata:
   name: "{{ .Release.Name }}-galley-integration"
   annotations:
-    "helm.sh/hook": test-success
+    "helm.sh/hook": test
   labels:
     app: galley-integration
     release: {{ .Release.Name }}
diff --git a/charts/galley/templates/tests/secret.yaml b/charts/galley/templates/tests/secret.yaml
index 74f118d1c2..ec0e5624b2 100644
--- a/charts/galley/templates/tests/secret.yaml
+++ b/charts/galley/templates/tests/secret.yaml
@@ -2,6 +2,8 @@ apiVersion: v1
 kind: ConfigMap
 metadata:
   name: galley-integration-secrets
+  annotations:
+    "helm.sh/hook": test
 data:
   # These "secrets" are only used in tests and are therefore safe to be stored unencrypted
   provider-privatekey.pem: |
diff --git a/charts/gundeck/templates/tests/configmap.yaml b/charts/gundeck/templates/tests/configmap.yaml
index 5c398d39e1..004d58f060 100644
--- a/charts/gundeck/templates/tests/configmap.yaml
+++ b/charts/gundeck/templates/tests/configmap.yaml
@@ -2,6 +2,8 @@ apiVersion: v1
 kind: ConfigMap
 metadata:
   name: "gundeck-integration"
+  annotations:
+    "helm.sh/hook": test
 data:
   integration.yaml: |
     gundeck:
diff --git a/charts/gundeck/templates/tests/gundeck-integration.yaml b/charts/gundeck/templates/tests/gundeck-integration.yaml
index 8424fd3770..4f81fa2c22 100644
--- a/charts/gundeck/templates/tests/gundeck-integration.yaml
+++ b/charts/gundeck/templates/tests/gundeck-integration.yaml
@@ -3,7 +3,7 @@ kind: Pod
 metadata:
   name: "{{ .Release.Name }}-gundeck-integration"
   annotations:
-    "helm.sh/hook": test-success
+    "helm.sh/hook": test
 spec:
   volumes:
     - name: "gundeck-integration"
diff --git a/charts/spar/templates/tests/configmap.yaml b/charts/spar/templates/tests/configmap.yaml
index e446ee7bdd..8773895447 100644
--- a/charts/spar/templates/tests/configmap.yaml
+++ b/charts/spar/templates/tests/configmap.yaml
@@ -2,6 +2,8 @@ apiVersion: v1
 kind: ConfigMap
 metadata:
   name: "spar-integration"
+  annotations:
+    "helm.sh/hook": test
 data:
   integration.yaml: |
     brig:
diff --git a/charts/spar/templates/tests/spar-integration.yaml b/charts/spar/templates/tests/spar-integration.yaml
index c4735ffd15..7063bbb745 100644
--- a/charts/spar/templates/tests/spar-integration.yaml
+++ b/charts/spar/templates/tests/spar-integration.yaml
@@ -3,7 +3,7 @@ kind: Pod
 metadata:
   name: "{{ .Release.Name }}-spar-integration"
   annotations:
-    "helm.sh/hook": test-success
+    "helm.sh/hook": test
   labels:
     app: spar-integration
     release: {{ .Release.Name }}
diff --git a/nix/overlay.nix b/nix/overlay.nix
index d63b121432..1f20335dbb 100644
--- a/nix/overlay.nix
+++ b/nix/overlay.nix
@@ -87,18 +87,7 @@ self: super: {
     inherit (super) stdenv fetchurl;
   };
 
-  helm = staticBinaryInTarball {
-    pname = "helm";
-    version = "3.6.3";
-
-    darwinAmd64Url = "https://get.helm.sh/helm-v3.6.3-darwin-amd64.tar.gz";
-    darwinAmd64Sha256 = "0djjvgla8cw27h8s4y6jby19f74j58byb2vfv590cd03vlbzz8c4";
-
-    linuxAmd64Url = "https://get.helm.sh/helm-v3.6.3-linux-amd64.tar.gz";
-    linuxAmd64Sha256 = "0qp28fq137b07haz4vsdbc5biagh60dcs29jj70ksqi5k6201h87";
-
-    inherit (super) stdenv fetchurl;
-  };
+  helm = super.callPackage ./pkgs/helm {};
 
   helmfile = staticBinary {
     pname = "helmfile";
diff --git a/nix/pkgs/helm/default.nix b/nix/pkgs/helm/default.nix
new file mode 100644
index 0000000000..8b68403913
--- /dev/null
+++ b/nix/pkgs/helm/default.nix
@@ -0,0 +1,52 @@
+# Copied from nixpkgs and modified because it seems too complicated to override
+# buildGoModule packages.
+{ lib, stdenv, buildGoModule, fetchFromGitHub, installShellFiles }:
+
+buildGoModule rec {
+  pname = "kubernetes-helm";
+  version = "3.11.0-patched";
+
+  src = fetchFromGitHub {
+    owner = "wireapp";
+    repo = "helm";
+    rev = "949de3195be5b3d21ed707da18ee3bcb2a9a2af8";
+    sha256 = "sha256-alyR6+gm7WEvFfJxHl9a0jpC3+457Kg6aRHcidA0RZg=";
+  };
+  vendorSha256 = "sha256-LRMDrBSl5EGQqQt5FUU4JJHqdwfYt5qsVpe76jUQBVI=";
+
+  subPackages = [ "cmd/helm" ];
+  ldflags = [
+    "-w"
+    "-s"
+    "-X helm.sh/helm/v3/internal/version.version=v${version}"
+    "-X helm.sh/helm/v3/internal/version.gitCommit=${src.rev}"
+  ];
+
+  preCheck = ''
+    # skipping version tests because they require dot git directory
+    substituteInPlace cmd/helm/version_test.go \
+      --replace "TestVersion" "SkipVersion"
+  '' + lib.optionalString stdenv.isLinux ''
+    # skipping plugin tests on linux
+    substituteInPlace cmd/helm/plugin_test.go \
+      --replace "TestPluginDynamicCompletion" "SkipPluginDynamicCompletion" \
+      --replace "TestLoadPlugins" "SkipLoadPlugins"
+    substituteInPlace cmd/helm/helm_test.go \
+      --replace "TestPluginExitCode" "SkipPluginExitCode"
+  '';
+
+  nativeBuildInputs = [ installShellFiles ];
+  postInstall = ''
+    $out/bin/helm completion bash > helm.bash
+    $out/bin/helm completion zsh > helm.zsh
+    installShellCompletion helm.{bash,zsh}
+  '';
+
+  meta = with lib; {
+    homepage = "https://github.com/kubernetes/helm";
+    description = "A package manager for kubernetes";
+    mainProgram = "helm";
+    license = licenses.asl20;
+    maintainers = with maintainers; [ rlupton20 edude03 saschagrunert Frostman Chili-Man techknowlogick ];
+  };
+}
diff --git a/nix/wire-server.nix b/nix/wire-server.nix
index 99dc0caaaf..33320d327d 100644
--- a/nix/wire-server.nix
+++ b/nix/wire-server.nix
@@ -308,6 +308,8 @@ let
     pkgs.ormolu
     pkgs.shellcheck
     pkgs.treefmt
+    pkgs.gawk
+    pkgs.cfssl
     (hlib.justStaticExecutables pkgs.haskellPackages.cabal-fmt)
   ] ++ pkgs.lib.optionals pkgs.stdenv.isLinux [
     pkgs.skopeo
@@ -365,7 +367,6 @@ in
     paths = commonTools ++ [
       (pkgs.haskell-language-server.override { supportedGhcVersions = [ "92" ]; })
       pkgs.ghcid
-      pkgs.cfssl
       pkgs.kind
       pkgs.netcat
       pkgs.niv