From 91c1cdb9974fdcc3e337825d25f86c46da491023 Mon Sep 17 00:00:00 2001
From: jschaul <jschaul@users.noreply.github.com>
Date: Thu, 2 Mar 2023 16:56:22 +0100
Subject: [PATCH 01/40] init WIP

---
 .../nginx-ingress-controller-new/Chart.yaml   |  4 +++
 .../requirements.yaml                         |  4 +++
 .../nginx-ingress-controller-new/values.yaml  | 36 +++++++++++++++++++
 3 files changed, 44 insertions(+)
 create mode 100644 charts/nginx-ingress-controller-new/Chart.yaml
 create mode 100644 charts/nginx-ingress-controller-new/requirements.yaml
 create mode 100644 charts/nginx-ingress-controller-new/values.yaml

diff --git a/charts/nginx-ingress-controller-new/Chart.yaml b/charts/nginx-ingress-controller-new/Chart.yaml
new file mode 100644
index 0000000000..9990e003e8
--- /dev/null
+++ b/charts/nginx-ingress-controller-new/Chart.yaml
@@ -0,0 +1,4 @@
+apiVersion: v2
+description: A Helm chart for an ingress controller (using nginx) on Kubernetes
+name: nginx-ingress-controller-new
+version: 0.0.42
diff --git a/charts/nginx-ingress-controller-new/requirements.yaml b/charts/nginx-ingress-controller-new/requirements.yaml
new file mode 100644
index 0000000000..972753357f
--- /dev/null
+++ b/charts/nginx-ingress-controller-new/requirements.yaml
@@ -0,0 +1,4 @@
+dependencies:
+- name: ingress-nginx
+  version: 4.5.2 # k8s compatibility [1.21 - 1.26]
+  repository: https://kubernetes.github.io/ingress-nginx
diff --git a/charts/nginx-ingress-controller-new/values.yaml b/charts/nginx-ingress-controller-new/values.yaml
new file mode 100644
index 0000000000..9fcf69724a
--- /dev/null
+++ b/charts/nginx-ingress-controller-new/values.yaml
@@ -0,0 +1,36 @@
+# the following defaults apply to an on-prem bare-metal setup in the same spirit as the
+# older similarly named wrapper chart 'nginx-ingress-controller' (note the swapped words
+# 'nginx' and 'ingress') We assume no load balancer support and instead expose NodePorts
+# on ports 31773 and 31772, assuming traffic gets to these ports in another way.
+#
+# See https://github.com/kubernetes/ingress-nginx/blob/main/charts/ingress-nginx/values.yaml for all possible values to override.
+ingress-nginx:
+  service:
+    type: NodePort  # or LoadBalancer
+    externalTrafficPolicy: Local
+    nodePorts:
+      # The nginx instance is exposed on ports 31773 (https) and 31772 (http)
+      # on the node on which it runs. You should add a port-forwarding rule
+      # on the node or on the loadbalancer that forwards ports 443 and 80 to
+      # these respective ports.
+      https: 31773
+      http: 31772
+
+  controller:
+    watchIngressWithoutClass: true
+    enableTopologyAwareRouting: true
+    # -- Use a `DaemonSet` or `Deployment`
+    kind: DaemonSet
+    config:
+      # NOTE: These are some sane defaults (compliant to TR-02102-2), you may want to overrride them on your own installation
+      # For TR-02102-2 see https://www.bsi.bund.de/SharedDocs/Downloads/EN/BSI/Publications/TechGuidelines/TG02102/BSI-TR-02102-2.html
+      # As a Wire employee, for Wire-internal discussions and context see
+      # * https://wearezeta.atlassian.net/browse/FS-33
+      # * https://wearezeta.atlassian.net/browse/FS-444
+      ssl-ciphers: "TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384"
+      ssl-protocols: "TLSv1.2 TLSv1.3"
+      # used to be called http2-max-(header|field)-size, removed in controller v1.3
+      large-client-header-buffers: "16 32k"
+      proxy-buffer-size: "16k"
+      proxy-body-size: "1024m"
+      hsts-max-age: "31536000"

From 09dfeb49a8ff34d343801560bcce8a99b3ef5e44 Mon Sep 17 00:00:00 2001
From: jschaul <jschaul@users.noreply.github.com>
Date: Thu, 2 Mar 2023 17:42:15 +0100
Subject: [PATCH 02/40] changelog

---
 changelog.d/0-release-notes/ingress-controller | 12 ++++++++++++
 1 file changed, 12 insertions(+)
 create mode 100644 changelog.d/0-release-notes/ingress-controller

diff --git a/changelog.d/0-release-notes/ingress-controller b/changelog.d/0-release-notes/ingress-controller
new file mode 100644
index 0000000000..1be448c4a6
--- /dev/null
+++ b/changelog.d/0-release-notes/ingress-controller
@@ -0,0 +1,12 @@
+New nginx-ingress-controller wrapper chart compatible with k8s versions [1.21 - 1.26].
+In case you have custom overrides, you need to modify the top-level configuration key:
+
+```diff
+# IF you have overrides in
+# nginx-ingress-controller/values.yaml
+-nginx-ingress:
++ingress-nginx:
+   controller:
+```
+
+and double-check if all overrides you use are indeed provided under the same name by the upstream chart.

From d3e85bf89971b034da585f37f8c1eb5cecdc28c4 Mon Sep 17 00:00:00 2001
From: jschaul <jschaul@users.noreply.github.com>
Date: Thu, 9 Mar 2023 15:42:30 +0100
Subject: [PATCH 03/40] more overrrides: log format, TLS 1.3 ciphers

---
 charts/nginx-ingress-controller-new/values.yaml | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/charts/nginx-ingress-controller-new/values.yaml b/charts/nginx-ingress-controller-new/values.yaml
index 9fcf69724a..5ae60cf35e 100644
--- a/charts/nginx-ingress-controller-new/values.yaml
+++ b/charts/nginx-ingress-controller-new/values.yaml
@@ -27,10 +27,20 @@ ingress-nginx:
       # As a Wire employee, for Wire-internal discussions and context see
       # * https://wearezeta.atlassian.net/browse/FS-33
       # * https://wearezeta.atlassian.net/browse/FS-444
-      ssl-ciphers: "TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384"
       ssl-protocols: "TLSv1.2 TLSv1.3"
+      # override cipher suites used in TLS 1.2 (only, if TLS 1.2 is used)
+      ssl-ciphers: "ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384"
+      # override cipher suites used in TLS 1.3 (only, if TLS 1.3 is used)
+      server-snippet: "ssl_conf_command Ciphersuites TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384;"
       # used to be called http2-max-(header|field)-size, removed in controller v1.3
       large-client-header-buffers: "16 32k"
       proxy-buffer-size: "16k"
       proxy-body-size: "1024m"
       hsts-max-age: "31536000"
+      # Override log format to remove logging access tokens:
+      # removes 'request_query: "$args"', since it can include '?access_token=...'
+      # (sometimes sent for assets and websocket establishments)
+      # We do not wish to log these (SEC-47)
+      # Also add ssl/tls protocol/cipher to gain some observability here (can we turn off TLS 1.2?)
+      log-format-escape-json: true
+      log-format-upstream: '{"bytes_sent": "$bytes_sent", "duration": "$request_time", "http_referrer": "$http_referer", "http_user_agent": "$http_user_agent", "method": "$request_method", "path": "$uri", "remote_addr": "$proxy_protocol_addr", "remote_user": "$remote_user", "request_id": "$req_id", "request_length": "$request_length", "request_proto": "$server_protocol", "request_time": "$request_time", "status": "$status", "time": "$time_iso8601", "tls_cipher": "$ssl_cipher", "tls_protocol": "$ssl_protocol", "vhost": "$host", "x_forwarded_for": "$proxy_add_x_forwarded_for"}'

From 91bccf2cc7a4a21b9c1306354185277e4d7d2e51 Mon Sep 17 00:00:00 2001
From: jschaul <jschaul@users.noreply.github.com>
Date: Thu, 9 Mar 2023 15:57:44 +0100
Subject: [PATCH 04/40] shorten names of installed helm charts

---
 hack/helmfile.yaml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/hack/helmfile.yaml b/hack/helmfile.yaml
index 6392d64a43..6adcaec90b 100644
--- a/hack/helmfile.yaml
+++ b/hack/helmfile.yaml
@@ -69,19 +69,19 @@ releases:
     values:
       - './helm_vars/redis-cluster/values.yaml.gotmpl'
 
-  - name: '{{ .Values.namespace }}-nginx-ingress-controller'
+  - name: '{{ .Values.namespace }}-ingress-c'
     namespace: '{{ .Values.namespace }}'
     chart: '../.local/charts/nginx-ingress-controller'
     values:
       - './helm_vars/nginx-ingress-controller/values.yaml'
 
-  - name: '{{ .Values.namespace }}-nginx-ingress-controller-2'
+  - name: '{{ .Values.namespace }}-ingress-c-2'
     namespace: '{{ .Values.namespaceFed2 }}'
     chart: '../.local/charts/nginx-ingress-controller'
     values:
       - './helm_vars/nginx-ingress-controller/values.yaml'
 
-  - name: '{{ .Values.namespace }}-nginx-ingress-services'
+  - name: '{{ .Values.namespace }}-ingresses'
     namespace: '{{ .Values.namespace }}'
     chart: '../.local/charts/nginx-ingress-services'
     values:
@@ -94,7 +94,7 @@ releases:
       - name: config.dns.federator
         value: {{ .Values.federationDomain }}
 
-  - name: '{{ .Values.namespace }}-nginx-ingress-services-2'
+  - name: '{{ .Values.namespace }}-ingresses-2'
     namespace: '{{ .Values.namespaceFed2 }}'
     chart: '../.local/charts/nginx-ingress-services'
     values:

From 11599e9d6baa5b99b492f7f7b8ff816a4c0eaafc Mon Sep 17 00:00:00 2001
From: jschaul <jschaul@users.noreply.github.com>
Date: Thu, 9 Mar 2023 16:00:35 +0100
Subject: [PATCH 05/40] rename, add to makefile releases

---
 Makefile                                                        | 2 +-
 .../Chart.yaml                                                  | 2 +-
 .../requirements.yaml                                           | 0
 .../values.yaml                                                 | 0
 4 files changed, 2 insertions(+), 2 deletions(-)
 rename charts/{nginx-ingress-controller-new => ingress-nginx-controller}/Chart.yaml (76%)
 rename charts/{nginx-ingress-controller-new => ingress-nginx-controller}/requirements.yaml (100%)
 rename charts/{nginx-ingress-controller-new => ingress-nginx-controller}/values.yaml (100%)

diff --git a/Makefile b/Makefile
index 20d1110d9c..fff197515c 100644
--- a/Makefile
+++ b/Makefile
@@ -17,7 +17,7 @@ CHARTS_RELEASE := wire-server redis-ephemeral redis-cluster databases-ephemeral
 fake-aws fake-aws-s3 fake-aws-sqs aws-ingress fluent-bit kibana backoffice		\
 calling-test demo-smtp elasticsearch-curator elasticsearch-external				\
 elasticsearch-ephemeral minio-external cassandra-external						\
-nginx-ingress-controller nginx-ingress-services reaper sftd restund coturn		\
+nginx-ingress-controller ingress-nginx-controller nginx-ingress-services reaper sftd restund coturn		\
 inbucket k8ssandra-test-cluster postgresql
 KIND_CLUSTER_NAME     := wire-server
 HELM_PARALLELISM      ?= 1 # 1 for sequential tests; 6 for all-parallel tests
diff --git a/charts/nginx-ingress-controller-new/Chart.yaml b/charts/ingress-nginx-controller/Chart.yaml
similarity index 76%
rename from charts/nginx-ingress-controller-new/Chart.yaml
rename to charts/ingress-nginx-controller/Chart.yaml
index 9990e003e8..2859f18374 100644
--- a/charts/nginx-ingress-controller-new/Chart.yaml
+++ b/charts/ingress-nginx-controller/Chart.yaml
@@ -1,4 +1,4 @@
 apiVersion: v2
 description: A Helm chart for an ingress controller (using nginx) on Kubernetes
-name: nginx-ingress-controller-new
+name: ingress-nginx-controller
 version: 0.0.42
diff --git a/charts/nginx-ingress-controller-new/requirements.yaml b/charts/ingress-nginx-controller/requirements.yaml
similarity index 100%
rename from charts/nginx-ingress-controller-new/requirements.yaml
rename to charts/ingress-nginx-controller/requirements.yaml
diff --git a/charts/nginx-ingress-controller-new/values.yaml b/charts/ingress-nginx-controller/values.yaml
similarity index 100%
rename from charts/nginx-ingress-controller-new/values.yaml
rename to charts/ingress-nginx-controller/values.yaml

From 2a4c0e96a0ace8f8ed3fa49784fbf2e9768a23af Mon Sep 17 00:00:00 2001
From: jschaul <jschaul@users.noreply.github.com>
Date: Mon, 13 Mar 2023 17:17:45 +0100
Subject: [PATCH 06/40] also add new chart to list of charts for integration
 tests

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index fff197515c..f32a11814e 100644
--- a/Makefile
+++ b/Makefile
@@ -7,7 +7,7 @@ DOCKER_TAG            ?= $(USER)
 # default helm chart version must be 0.0.42 for local development (because 42 is the answer to the universe and everything)
 HELM_SEMVER           ?= 0.0.42
 # The list of helm charts needed on internal kubernetes testing environments
-CHARTS_INTEGRATION    := wire-server databases-ephemeral redis-cluster fake-aws nginx-ingress-controller nginx-ingress-services fluent-bit kibana sftd restund coturn
+CHARTS_INTEGRATION    := wire-server databases-ephemeral redis-cluster fake-aws ingress-nginx-controller nginx-ingress-controller nginx-ingress-services fluent-bit kibana sftd restund coturn
 # The list of helm charts to publish on S3
 # FUTUREWORK: after we "inline local subcharts",
 # (e.g. move charts/brig to charts/wire-server/brig)

From 1def7e535e15d940a5b6887b1644185ef6a319d9 Mon Sep 17 00:00:00 2001
From: jschaul <jschaul@users.noreply.github.com>
Date: Mon, 13 Mar 2023 17:21:21 +0100
Subject: [PATCH 07/40] move dependencies to Chart.yaml

---
 charts/ingress-nginx-controller/Chart.yaml        | 4 ++++
 charts/ingress-nginx-controller/requirements.yaml | 4 ----
 2 files changed, 4 insertions(+), 4 deletions(-)
 delete mode 100644 charts/ingress-nginx-controller/requirements.yaml

diff --git a/charts/ingress-nginx-controller/Chart.yaml b/charts/ingress-nginx-controller/Chart.yaml
index 2859f18374..615079ed82 100644
--- a/charts/ingress-nginx-controller/Chart.yaml
+++ b/charts/ingress-nginx-controller/Chart.yaml
@@ -2,3 +2,7 @@ apiVersion: v2
 description: A Helm chart for an ingress controller (using nginx) on Kubernetes
 name: ingress-nginx-controller
 version: 0.0.42
+dependencies:
+- name: ingress-nginx
+  version: 4.5.2 # k8s compatibility [1.21 - 1.26]
+  repository: https://kubernetes.github.io/ingress-nginx
diff --git a/charts/ingress-nginx-controller/requirements.yaml b/charts/ingress-nginx-controller/requirements.yaml
deleted file mode 100644
index 972753357f..0000000000
--- a/charts/ingress-nginx-controller/requirements.yaml
+++ /dev/null
@@ -1,4 +0,0 @@
-dependencies:
-- name: ingress-nginx
-  version: 4.5.2 # k8s compatibility [1.21 - 1.26]
-  repository: https://kubernetes.github.io/ingress-nginx

From 7c05c5831c5f8ecda08509a1f64bf54a24164ad4 Mon Sep 17 00:00:00 2001
From: jschaul <jschaul@users.noreply.github.com>
Date: Mon, 13 Mar 2023 17:37:58 +0100
Subject: [PATCH 08/40] also download dependencies of helm charts if specified
 inside Chart.yaml

---
 hack/bin/set-helm-chart-version.sh | 2 +-
 hack/bin/update.sh                 | 6 ++++++
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/hack/bin/set-helm-chart-version.sh b/hack/bin/set-helm-chart-version.sh
index 759da9a218..00b838642e 100755
--- a/hack/bin/set-helm-chart-version.sh
+++ b/hack/bin/set-helm-chart-version.sh
@@ -12,7 +12,7 @@ tempfile=$(mktemp)
 
 function update_chart(){
     chart_file=$1
-    sed -e "s/version: .*/version: $target_version/g" "$chart_file" > "$tempfile" && mv "$tempfile" "$chart_file"
+    sed -e "s/^version: .*/version: $target_version/g" "$chart_file" > "$tempfile" && mv "$tempfile" "$chart_file"
 }
 
 function write_versions() {
diff --git a/hack/bin/update.sh b/hack/bin/update.sh
index 20d597a796..47ecefb2b0 100755
--- a/hack/bin/update.sh
+++ b/hack/bin/update.sh
@@ -29,6 +29,12 @@ helmDepUp () {
       helm dep up
       echo "... updating in $path done."
     fi
+
+    if grep "dependencies:" Chart.yaml; then
+      echo "Updating dependencies (from Chart.yaml) in $path ..."
+      helm dep up
+      echo "... updating in $path done."
+    fi
 }
 
 helmDepUp "$dir"

From f3ab9428aa124c24c3aa9520817cf8566ba4c62d Mon Sep 17 00:00:00 2001
From: jschaul <jschaul@users.noreply.github.com>
Date: Mon, 13 Mar 2023 18:01:42 +0100
Subject: [PATCH 09/40] update comment

---
 charts/ingress-nginx-controller/Chart.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/charts/ingress-nginx-controller/Chart.yaml b/charts/ingress-nginx-controller/Chart.yaml
index 615079ed82..479a9b5d90 100644
--- a/charts/ingress-nginx-controller/Chart.yaml
+++ b/charts/ingress-nginx-controller/Chart.yaml
@@ -4,5 +4,5 @@ name: ingress-nginx-controller
 version: 0.0.42
 dependencies:
 - name: ingress-nginx
-  version: 4.5.2 # k8s compatibility [1.21 - 1.26]
+  version: 4.5.2 # k8s compatibility [1.23 - 1.26]
   repository: https://kubernetes.github.io/ingress-nginx

From f7238a1ce180b00896d851d7bcebb19c0dc1d643 Mon Sep 17 00:00:00 2001
From: jschaul <jschaul@users.noreply.github.com>
Date: Mon, 13 Mar 2023 19:00:40 +0100
Subject: [PATCH 10/40] overrides

---
 hack/bin/integration-setup-federation.sh         |  2 +-
 hack/bin/integration-setup.sh                    |  2 +-
 .../ingress-nginx-controller/values.yaml.gotmpl  | 16 ++++++++++++++++
 3 files changed, 18 insertions(+), 2 deletions(-)
 create mode 100644 hack/helm_vars/ingress-nginx-controller/values.yaml.gotmpl

diff --git a/hack/bin/integration-setup-federation.sh b/hack/bin/integration-setup-federation.sh
index f569928239..2161053af9 100755
--- a/hack/bin/integration-setup-federation.sh
+++ b/hack/bin/integration-setup-federation.sh
@@ -20,7 +20,7 @@ ${DIR}/integration-cleanup.sh
 # script beforehand on all relevant charts to download the nested dependencies
 # (e.g. cassandra from underneath databases-ephemeral)
 echo "updating recursive dependencies ..."
-charts=(fake-aws databases-ephemeral redis-cluster wire-server nginx-ingress-controller nginx-ingress-services)
+charts=(fake-aws databases-ephemeral redis-cluster wire-server ingress-nginx-controller nginx-ingress-controller nginx-ingress-services)
 mkdir -p ~/.parallel && touch ~/.parallel/will-cite
 printf '%s\n' "${charts[@]}" | parallel -P "${HELM_PARALLELISM}" "$DIR/update.sh" "$CHARTS_DIR/{}"
 
diff --git a/hack/bin/integration-setup.sh b/hack/bin/integration-setup.sh
index 59cf0e4f84..ab878ff79d 100755
--- a/hack/bin/integration-setup.sh
+++ b/hack/bin/integration-setup.sh
@@ -14,7 +14,7 @@ HELM_PARALLELISM=${HELM_PARALLELISM:-1}
 "${DIR}/integration-cleanup.sh"
 
 echo "updating recursive dependencies ..."
-charts=(fake-aws databases-ephemeral redis-cluster wire-server nginx-ingress-controller nginx-ingress-services)
+charts=(fake-aws databases-ephemeral redis-cluster wire-server ingress-nginx-controller nginx-ingress-controller nginx-ingress-services)
 mkdir -p ~/.parallel && touch ~/.parallel/will-cite
 printf '%s\n' "${charts[@]}" | parallel -P "${HELM_PARALLELISM}" "$DIR/update.sh" "$CHARTS_DIR/{}"
 
diff --git a/hack/helm_vars/ingress-nginx-controller/values.yaml.gotmpl b/hack/helm_vars/ingress-nginx-controller/values.yaml.gotmpl
new file mode 100644
index 0000000000..fe5d943f6a
--- /dev/null
+++ b/hack/helm_vars/ingress-nginx-controller/values.yaml.gotmpl
@@ -0,0 +1,16 @@
+ingress-nginx:
+  controller:
+    ingressClassResource:
+        name: "nginx-{{ .Release.Namespace }}"
+        # -- Is this ingressClass enabled or not
+        enabled: true
+    ingressClass: "nginx-{{ .Release.Namespace }}"
+    kind: Deployment
+    replicaCount: 1
+    service:
+      type: NodePort
+      externalTrafficPolicy: Local
+      nodePorts:
+        # choose a random free port
+        https: null
+        http: null

From 018290e9552b74f9f31bfbcf85699ab0f15422aa Mon Sep 17 00:00:00 2001
From: jschaul <jschaul@users.noreply.github.com>
Date: Mon, 13 Mar 2023 19:05:04 +0100
Subject: [PATCH 11/40] move overrides to correct location

---
 charts/ingress-nginx-controller/values.yaml   | 21 +++++++++----------
 .../values.yaml.gotmpl                        |  2 --
 2 files changed, 10 insertions(+), 13 deletions(-)

diff --git a/charts/ingress-nginx-controller/values.yaml b/charts/ingress-nginx-controller/values.yaml
index 5ae60cf35e..1b796d54a6 100644
--- a/charts/ingress-nginx-controller/values.yaml
+++ b/charts/ingress-nginx-controller/values.yaml
@@ -5,22 +5,21 @@
 #
 # See https://github.com/kubernetes/ingress-nginx/blob/main/charts/ingress-nginx/values.yaml for all possible values to override.
 ingress-nginx:
-  service:
-    type: NodePort  # or LoadBalancer
-    externalTrafficPolicy: Local
-    nodePorts:
-      # The nginx instance is exposed on ports 31773 (https) and 31772 (http)
-      # on the node on which it runs. You should add a port-forwarding rule
-      # on the node or on the loadbalancer that forwards ports 443 and 80 to
-      # these respective ports.
-      https: 31773
-      http: 31772
-
   controller:
     watchIngressWithoutClass: true
     enableTopologyAwareRouting: true
     # -- Use a `DaemonSet` or `Deployment`
     kind: DaemonSet
+    service:
+      type: NodePort  # or LoadBalancer
+      externalTrafficPolicy: Local
+      nodePorts:
+        # The nginx instance is exposed on ports 31773 (https) and 31772 (http)
+        # on the node on which it runs. You should add a port-forwarding rule
+        # on the node or on the loadbalancer that forwards ports 443 and 80 to
+        # these respective ports.
+        https: 31773
+        http: 31772
     config:
       # NOTE: These are some sane defaults (compliant to TR-02102-2), you may want to overrride them on your own installation
       # For TR-02102-2 see https://www.bsi.bund.de/SharedDocs/Downloads/EN/BSI/Publications/TechGuidelines/TG02102/BSI-TR-02102-2.html
diff --git a/hack/helm_vars/ingress-nginx-controller/values.yaml.gotmpl b/hack/helm_vars/ingress-nginx-controller/values.yaml.gotmpl
index fe5d943f6a..eea4a26a83 100644
--- a/hack/helm_vars/ingress-nginx-controller/values.yaml.gotmpl
+++ b/hack/helm_vars/ingress-nginx-controller/values.yaml.gotmpl
@@ -8,8 +8,6 @@ ingress-nginx:
     kind: Deployment
     replicaCount: 1
     service:
-      type: NodePort
-      externalTrafficPolicy: Local
       nodePorts:
         # choose a random free port
         https: null

From f7693b93378e918eea8acafc46735a55d9a1071d Mon Sep 17 00:00:00 2001
From: jschaul <jschaul@users.noreply.github.com>
Date: Mon, 13 Mar 2023 19:09:54 +0100
Subject: [PATCH 12/40] switch helmfile over to new ingress for testing...

---
 hack/helmfile.yaml | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/hack/helmfile.yaml b/hack/helmfile.yaml
index 6adcaec90b..cec7333f2b 100644
--- a/hack/helmfile.yaml
+++ b/hack/helmfile.yaml
@@ -36,6 +36,9 @@ repositories:
   - name: bitnami
     url: 'https://charts.bitnami.com/bitnami'
 
+  - name: ingress
+    url: https://kubernetes.github.io/ingress-nginx
+
 releases:
   - name: '{{ .Values.namespace }}-fake-aws'
     namespace: '{{ .Values.namespace }}'
@@ -71,15 +74,15 @@ releases:
 
   - name: '{{ .Values.namespace }}-ingress-c'
     namespace: '{{ .Values.namespace }}'
-    chart: '../.local/charts/nginx-ingress-controller'
+    chart: '../.local/charts/ingress-nginx-controller'
     values:
-      - './helm_vars/nginx-ingress-controller/values.yaml'
+      - './helm_vars/ingress-nginx-controller/values.yaml.gotmpl'
 
   - name: '{{ .Values.namespace }}-ingress-c-2'
     namespace: '{{ .Values.namespaceFed2 }}'
-    chart: '../.local/charts/nginx-ingress-controller'
+    chart: '../.local/charts/ingress-nginx-controller'
     values:
-      - './helm_vars/nginx-ingress-controller/values.yaml'
+      - './helm_vars/ingress-nginx-controller/values.yaml.gotmpl'
 
   - name: '{{ .Values.namespace }}-ingresses'
     namespace: '{{ .Values.namespace }}'
@@ -107,8 +110,6 @@ releases:
       - name: config.dns.federator
         value: {{ .Values.federationDomainFed2 }}
 
-  #---------------------------------------------
-  #
   # Note that wire-server depends on databases-ephemeral being up; and in some
   # cases on nginx-ingress also being up. If installing helm charts in a
   # parallel way, it's expected to see some wire-server pods (namely the

From a601a4c79692f493106ea81c2d9ea1f766340f53 Mon Sep 17 00:00:00 2001
From: jschaul <jschaul@users.noreply.github.com>
Date: Mon, 13 Mar 2023 20:05:27 +0100
Subject: [PATCH 13/40] update changelog

---
 .../0-release-notes/ingress-controller        | 20 ++++++++++++++-----
 1 file changed, 15 insertions(+), 5 deletions(-)

diff --git a/changelog.d/0-release-notes/ingress-controller b/changelog.d/0-release-notes/ingress-controller
index 1be448c4a6..0d381e6815 100644
--- a/changelog.d/0-release-notes/ingress-controller
+++ b/changelog.d/0-release-notes/ingress-controller
@@ -1,12 +1,22 @@
-New nginx-ingress-controller wrapper chart compatible with k8s versions [1.21 - 1.26].
-In case you have custom overrides, you need to modify the top-level configuration key:
+New 'ingress-nginx-controller' wrapper chart compatible with k8s versions [1.23 - 1.26]. The old one 'nginx-ingress-controller' (compatible only up to k8s 1.19) is now DEPRECATED.
+We advise to upgrade your version of kubernetes in use, and to make use of the new ingress controller chart. Main features:
+- up-to-date nginx version ('1.21.6')
+- TLS 1.3 support (including allowing specifying which cipher suites to use)
+- security fixes
+- no more accidental logging of Wire access tokens under specific circumstances
+
+The 'kind: Ingress' resources installed via 'nginx-ingress-services' chart remain compatible with both the old and the new ingress controller, and k8s versions [1.18 - 1.26]. In case you upgrade an existing kubernetes cluster (not recommended), you may need to first uninstall the old controller before installing the new controller chart.
+
+In case you have custom overrides, you need to modify the directory name and top-level configuration key:
 
 ```diff
-# IF you have overrides in
-# nginx-ingress-controller/values.yaml
+# If you have overrides for the controller chart (such as cipher suites), ensure to rename file and top-level key:
+-# nginx-ingress-controller/values.yaml
++# ingress-nginx-controller/values.yaml
 -nginx-ingress:
 +ingress-nginx:
    controller:
+     # ...
 ```
 
-and double-check if all overrides you use are indeed provided under the same name by the upstream chart.
+and double-check if all overrides you use are indeed provided under the same name by the upstream chart. See also the default overrides in [the default values.yaml](https://github.com/wireapp/wire-server/blob/develop/charts/ingress-nginx-controller/values.yaml).

From 744b9aacb8e9c52b8c9a786bd342910f2b08c093 Mon Sep 17 00:00:00 2001
From: jschaul <jschaul@users.noreply.github.com>
Date: Mon, 13 Mar 2023 20:15:06 +0100
Subject: [PATCH 14/40] add deprecated comment to old chart

---
 charts/nginx-ingress-controller/Chart.yaml | 3 ++-
 charts/nginx-ingress-controller/README.md  | 2 ++
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/charts/nginx-ingress-controller/Chart.yaml b/charts/nginx-ingress-controller/Chart.yaml
index 96f94a8958..3177840c9c 100644
--- a/charts/nginx-ingress-controller/Chart.yaml
+++ b/charts/nginx-ingress-controller/Chart.yaml
@@ -1,4 +1,5 @@
 apiVersion: v1
-description: A Helm chart for an ingress controller (using nginx) on Kubernetes
+description: ingress-controller. DEPRECATED. Use ingress-nginx-controller chart instead.
 name: nginx-ingress-controller
 version: 0.0.42
+deprecated: true
diff --git a/charts/nginx-ingress-controller/README.md b/charts/nginx-ingress-controller/README.md
index 11a8508044..b0a4d744b1 100644
--- a/charts/nginx-ingress-controller/README.md
+++ b/charts/nginx-ingress-controller/README.md
@@ -1,3 +1,5 @@
+WARNING: deprecated. Use ingress-nginx-controller instead, if possible.
+
 This deploys a single ingress controller - ideally, you want this on a separate, shared namespace since controllers listen on all namespaces by default (you can also modify that but it's generally discouraged).
 
 It is mostly a wrapper of the [nginx-ingress](https://github.com/helm/charts/blob/master/stable/nginx-ingress/README.md) with some other defaults that make sense for our use case(s).

From ce294c3369cc9d91beee52a591fef5ca2bc8aef0 Mon Sep 17 00:00:00 2001
From: jschaul <jschaul@users.noreply.github.com>
Date: Wed, 15 Mar 2023 13:22:22 +0100
Subject: [PATCH 15/40] try out conditionals in helm chart

---
 hack/helmfile.yaml | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/hack/helmfile.yaml b/hack/helmfile.yaml
index cec7333f2b..cb8fe9e373 100644
--- a/hack/helmfile.yaml
+++ b/hack/helmfile.yaml
@@ -75,15 +75,33 @@ releases:
   - name: '{{ .Values.namespace }}-ingress-c'
     namespace: '{{ .Values.namespace }}'
     chart: '../.local/charts/ingress-nginx-controller'
+    condition: semverCompare ">= 1.23-0" .Capabilities.KubeVersion.Version
     values:
       - './helm_vars/ingress-nginx-controller/values.yaml.gotmpl'
 
   - name: '{{ .Values.namespace }}-ingress-c-2'
     namespace: '{{ .Values.namespaceFed2 }}'
     chart: '../.local/charts/ingress-nginx-controller'
+    condition: semverCompare ">= 1.23-0" .Capabilities.KubeVersion.Version
     values:
       - './helm_vars/ingress-nginx-controller/values.yaml.gotmpl'
 
+  # on old clusters, install the old controller chart
+  - name: '{{ .Values.namespace }}-ingress-co'
+    namespace: '{{ .Values.namespace }}'
+    chart: '../.local/charts/nginx-ingress-controller'
+    condition: semverCompare "<= 1.20-0" .Capabilities.KubeVersion.Version
+    values:
+      - './helm_vars/nginx-ingress-controller/values.yaml.gotmpl'
+
+  # on old clusters, install the old controller chart
+  - name: '{{ .Values.namespace }}-ingress-co-2'
+    namespace: '{{ .Values.namespaceFed2 }}'
+    chart: '../.local/charts/nginx-ingress-controller'
+    condition: semverCompare "<= 1.20-0" .Capabilities.KubeVersion.Version
+    values:
+      - './helm_vars/nginx-ingress-controller/values.yaml.gotmpl'
+
   - name: '{{ .Values.namespace }}-ingresses'
     namespace: '{{ .Values.namespace }}'
     chart: '../.local/charts/nginx-ingress-services'

From 281dea9577d7a3229e694b8ac591fe35f95e3d7c Mon Sep 17 00:00:00 2001
From: jschaul <jschaul@users.noreply.github.com>
Date: Wed, 15 Mar 2023 15:53:42 +0100
Subject: [PATCH 16/40] add kubernetes version in manually into helmfile

---
 hack/bin/integration-setup-federation.sh |  3 +++
 hack/bin/integration-setup.sh            |  3 +++
 hack/helmfile.yaml                       | 12 +++++++-----
 3 files changed, 13 insertions(+), 5 deletions(-)

diff --git a/hack/bin/integration-setup-federation.sh b/hack/bin/integration-setup-federation.sh
index 2161053af9..c6d6580e9e 100755
--- a/hack/bin/integration-setup-federation.sh
+++ b/hack/bin/integration-setup-federation.sh
@@ -27,6 +27,9 @@ printf '%s\n' "${charts[@]}" | parallel -P "${HELM_PARALLELISM}" "$DIR/update.sh
 # FUTUREWORK: use helm functions instead, see https://wearezeta.atlassian.net/browse/SQPIT-723
 echo "Generating self-signed certificates..."
 
+KUBERNETES_VERSION_MAJOR="$(kubectl version -o json | jq .serverVersion.major)"
+KUBERNETES_VERSION_MINOR="$(kubectl version -o json | jq .serverVersion.minor)"
+export KUBERNETES_VERSION="$KUBERNETES_VERSION_MAJOR.$KUBERNETES_VERSION_MINOR"
 export NAMESPACE_1="$NAMESPACE"
 export FEDERATION_DOMAIN_BASE="$NAMESPACE_1.svc.cluster.local"
 export FEDERATION_DOMAIN_1="federation-test-helper.$FEDERATION_DOMAIN_BASE"
diff --git a/hack/bin/integration-setup.sh b/hack/bin/integration-setup.sh
index ab878ff79d..96b715b6a5 100755
--- a/hack/bin/integration-setup.sh
+++ b/hack/bin/integration-setup.sh
@@ -18,6 +18,9 @@ charts=(fake-aws databases-ephemeral redis-cluster wire-server ingress-nginx-con
 mkdir -p ~/.parallel && touch ~/.parallel/will-cite
 printf '%s\n' "${charts[@]}" | parallel -P "${HELM_PARALLELISM}" "$DIR/update.sh" "$CHARTS_DIR/{}"
 
+KUBERNETES_VERSION_MAJOR="$(kubectl version -o json | jq .serverVersion.major)"
+KUBERNETES_VERSION_MINOR="$(kubectl version -o json | jq .serverVersion.minor)"
+export KUBERNETES_VERSION="$KUBERNETES_VERSION_MAJOR.$KUBERNETES_VERSION_MINOR"
 echo "Generating self-signed certificates..."
 export FEDERATION_DOMAIN_BASE="$NAMESPACE.svc.cluster.local"
 export FEDERATION_DOMAIN="federation-test-helper.$FEDERATION_DOMAIN_BASE"
diff --git a/hack/helmfile.yaml b/hack/helmfile.yaml
index cb8fe9e373..79d0f67539 100644
--- a/hack/helmfile.yaml
+++ b/hack/helmfile.yaml
@@ -18,6 +18,7 @@ environments:
       - federationDomain: {{ requiredEnv "FEDERATION_DOMAIN_1" }}
       - namespaceFed2: {{ requiredEnv "NAMESPACE_2" }}
       - federationDomainFed2: {{ requiredEnv "FEDERATION_DOMAIN_2" }}
+      - kubeVersion: {{ requiredEnv "KUBERNETES_VERSION" }}
       - imagePullPolicy: Always
       - redisStorageClass: hcloud-volumes
   kind:
@@ -26,6 +27,7 @@ environments:
       - federationDomain: {{ requiredEnv "FEDERATION_DOMAIN_1" }}
       - namespaceFed2: {{ requiredEnv "NAMESPACE_2" }}
       - federationDomainFed2: {{ requiredEnv "FEDERATION_DOMAIN_2" }}
+      - kubeVersion: {{ requiredEnv "KUBERNETES_VERSION" }}
       - imagePullPolicy: Never
       - redisStorageClass: standard
 
@@ -75,22 +77,22 @@ releases:
   - name: '{{ .Values.namespace }}-ingress-c'
     namespace: '{{ .Values.namespace }}'
     chart: '../.local/charts/ingress-nginx-controller'
-    condition: semverCompare ">= 1.23-0" .Capabilities.KubeVersion.Version
+    condition: {{ semverCompare ">= 1.23-0" .Values.kubeVersion }}
     values:
       - './helm_vars/ingress-nginx-controller/values.yaml.gotmpl'
 
   - name: '{{ .Values.namespace }}-ingress-c-2'
     namespace: '{{ .Values.namespaceFed2 }}'
     chart: '../.local/charts/ingress-nginx-controller'
-    condition: semverCompare ">= 1.23-0" .Capabilities.KubeVersion.Version
-    values:
+    condition: {{ semverCompare ">= 1.23-0" .Values.kubeVersion }}
+values:
       - './helm_vars/ingress-nginx-controller/values.yaml.gotmpl'
 
   # on old clusters, install the old controller chart
   - name: '{{ .Values.namespace }}-ingress-co'
     namespace: '{{ .Values.namespace }}'
     chart: '../.local/charts/nginx-ingress-controller'
-    condition: semverCompare "<= 1.20-0" .Capabilities.KubeVersion.Version
+    condition: {{ semverCompare "<= 1.20-0" .Values.kubeVersion }}
     values:
       - './helm_vars/nginx-ingress-controller/values.yaml.gotmpl'
 
@@ -98,7 +100,7 @@ releases:
   - name: '{{ .Values.namespace }}-ingress-co-2'
     namespace: '{{ .Values.namespaceFed2 }}'
     chart: '../.local/charts/nginx-ingress-controller'
-    condition: semverCompare "<= 1.20-0" .Capabilities.KubeVersion.Version
+    condition: {{ semverCompare "<= 1.20-0" .Values.kubeVersion }}
     values:
       - './helm_vars/nginx-ingress-controller/values.yaml.gotmpl'
 

From b4f50b7c0937d341fb16bc2ea463bdc27cab3bb3 Mon Sep 17 00:00:00 2001
From: jschaul <jschaul@users.noreply.github.com>
Date: Wed, 15 Mar 2023 16:05:04 +0100
Subject: [PATCH 17/40] ...

---
 hack/helmfile.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hack/helmfile.yaml b/hack/helmfile.yaml
index 79d0f67539..0d66ee158f 100644
--- a/hack/helmfile.yaml
+++ b/hack/helmfile.yaml
@@ -85,7 +85,7 @@ releases:
     namespace: '{{ .Values.namespaceFed2 }}'
     chart: '../.local/charts/ingress-nginx-controller'
     condition: {{ semverCompare ">= 1.23-0" .Values.kubeVersion }}
-values:
+    values:
       - './helm_vars/ingress-nginx-controller/values.yaml.gotmpl'
 
   # on old clusters, install the old controller chart

From 6f6f7df8ebc95f99f379fd981064f1e290b12d7c Mon Sep 17 00:00:00 2001
From: jschaul <jschaul@users.noreply.github.com>
Date: Wed, 15 Mar 2023 17:07:55 +0100
Subject: [PATCH 18/40] fixup

---
 hack/bin/integration-setup-federation.sh |  5 +++--
 hack/bin/integration-setup.sh            |  5 +++--
 hack/helmfile.yaml                       | 10 +++++-----
 3 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/hack/bin/integration-setup-federation.sh b/hack/bin/integration-setup-federation.sh
index c6d6580e9e..6a3a4f7d3a 100755
--- a/hack/bin/integration-setup-federation.sh
+++ b/hack/bin/integration-setup-federation.sh
@@ -27,9 +27,10 @@ printf '%s\n' "${charts[@]}" | parallel -P "${HELM_PARALLELISM}" "$DIR/update.sh
 # FUTUREWORK: use helm functions instead, see https://wearezeta.atlassian.net/browse/SQPIT-723
 echo "Generating self-signed certificates..."
 
-KUBERNETES_VERSION_MAJOR="$(kubectl version -o json | jq .serverVersion.major)"
-KUBERNETES_VERSION_MINOR="$(kubectl version -o json | jq .serverVersion.minor)"
+KUBERNETES_VERSION_MAJOR="$(kubectl version -o json | jq -r .serverVersion.major)"
+KUBERNETES_VERSION_MINOR="$(kubectl version -o json | jq -r .serverVersion.minor)"
 export KUBERNETES_VERSION="$KUBERNETES_VERSION_MAJOR.$KUBERNETES_VERSION_MINOR"
+echo "kubeVersion: $KUBERNETES_VERSION"
 export NAMESPACE_1="$NAMESPACE"
 export FEDERATION_DOMAIN_BASE="$NAMESPACE_1.svc.cluster.local"
 export FEDERATION_DOMAIN_1="federation-test-helper.$FEDERATION_DOMAIN_BASE"
diff --git a/hack/bin/integration-setup.sh b/hack/bin/integration-setup.sh
index 96b715b6a5..e48328795b 100755
--- a/hack/bin/integration-setup.sh
+++ b/hack/bin/integration-setup.sh
@@ -18,9 +18,10 @@ charts=(fake-aws databases-ephemeral redis-cluster wire-server ingress-nginx-con
 mkdir -p ~/.parallel && touch ~/.parallel/will-cite
 printf '%s\n' "${charts[@]}" | parallel -P "${HELM_PARALLELISM}" "$DIR/update.sh" "$CHARTS_DIR/{}"
 
-KUBERNETES_VERSION_MAJOR="$(kubectl version -o json | jq .serverVersion.major)"
-KUBERNETES_VERSION_MINOR="$(kubectl version -o json | jq .serverVersion.minor)"
+KUBERNETES_VERSION_MAJOR="$(kubectl version -o json | jq -r .serverVersion.major)"
+KUBERNETES_VERSION_MINOR="$(kubectl version -o json | jq -r .serverVersion.minor)"
 export KUBERNETES_VERSION="$KUBERNETES_VERSION_MAJOR.$KUBERNETES_VERSION_MINOR"
+echo "kubeVersion: $KUBERNETES_VERSION"
 echo "Generating self-signed certificates..."
 export FEDERATION_DOMAIN_BASE="$NAMESPACE.svc.cluster.local"
 export FEDERATION_DOMAIN="federation-test-helper.$FEDERATION_DOMAIN_BASE"
diff --git a/hack/helmfile.yaml b/hack/helmfile.yaml
index 0d66ee158f..34781dd4f2 100644
--- a/hack/helmfile.yaml
+++ b/hack/helmfile.yaml
@@ -18,7 +18,7 @@ environments:
       - federationDomain: {{ requiredEnv "FEDERATION_DOMAIN_1" }}
       - namespaceFed2: {{ requiredEnv "NAMESPACE_2" }}
       - federationDomainFed2: {{ requiredEnv "FEDERATION_DOMAIN_2" }}
-      - kubeVersion: {{ requiredEnv "KUBERNETES_VERSION" }}
+      - kubeVersion: {{ requiredEnv "KUBERNETES_VERSION" | quote }}
       - imagePullPolicy: Always
       - redisStorageClass: hcloud-volumes
   kind:
@@ -27,7 +27,7 @@ environments:
       - federationDomain: {{ requiredEnv "FEDERATION_DOMAIN_1" }}
       - namespaceFed2: {{ requiredEnv "NAMESPACE_2" }}
       - federationDomainFed2: {{ requiredEnv "FEDERATION_DOMAIN_2" }}
-      - kubeVersion: {{ requiredEnv "KUBERNETES_VERSION" }}
+      - kubeVersion: {{ requiredEnv "KUBERNETES_VERSION" | quote }}
       - imagePullPolicy: Never
       - redisStorageClass: standard
 
@@ -39,7 +39,7 @@ repositories:
     url: 'https://charts.bitnami.com/bitnami'
 
   - name: ingress
-    url: https://kubernetes.github.io/ingress-nginx
+    url: 'https://kubernetes.github.io/ingress-nginx'
 
 releases:
   - name: '{{ .Values.namespace }}-fake-aws'
@@ -94,7 +94,7 @@ releases:
     chart: '../.local/charts/nginx-ingress-controller'
     condition: {{ semverCompare "<= 1.20-0" .Values.kubeVersion }}
     values:
-      - './helm_vars/nginx-ingress-controller/values.yaml.gotmpl'
+      - './helm_vars/nginx-ingress-controller/values.yaml'
 
   # on old clusters, install the old controller chart
   - name: '{{ .Values.namespace }}-ingress-co-2'
@@ -102,7 +102,7 @@ releases:
     chart: '../.local/charts/nginx-ingress-controller'
     condition: {{ semverCompare "<= 1.20-0" .Values.kubeVersion }}
     values:
-      - './helm_vars/nginx-ingress-controller/values.yaml.gotmpl'
+      - './helm_vars/nginx-ingress-controller/values.yaml'
 
   - name: '{{ .Values.namespace }}-ingresses'
     namespace: '{{ .Values.namespace }}'

From 3fe920b2921ab30c13eea29e678d7b117d4bb232 Mon Sep 17 00:00:00 2001
From: jschaul <jschaul@users.noreply.github.com>
Date: Wed, 15 Mar 2023 18:02:33 +0100
Subject: [PATCH 19/40] ...

---
 hack/helmfile.yaml | 13 +++++++------
 nix/overlay.nix    | 15 ---------------
 2 files changed, 7 insertions(+), 21 deletions(-)

diff --git a/hack/helmfile.yaml b/hack/helmfile.yaml
index 34781dd4f2..1fd1fddd8c 100644
--- a/hack/helmfile.yaml
+++ b/hack/helmfile.yaml
@@ -18,7 +18,8 @@ environments:
       - federationDomain: {{ requiredEnv "FEDERATION_DOMAIN_1" }}
       - namespaceFed2: {{ requiredEnv "NAMESPACE_2" }}
       - federationDomainFed2: {{ requiredEnv "FEDERATION_DOMAIN_2" }}
-      - kubeVersion: {{ requiredEnv "KUBERNETES_VERSION" | quote }}
+      - kubeVersionAbove1_23: {{ semverCompare ">= 1.23-0" (requiredEnv "KUBERNETES_VERSION" | quote) }}
+      - kubeVersionBelow1_21: {{ semverCompare "< 1.21-0" (requiredEnv "KUBERNETES_VERSION" | quote) }}
       - imagePullPolicy: Always
       - redisStorageClass: hcloud-volumes
   kind:
@@ -27,7 +28,7 @@ environments:
       - federationDomain: {{ requiredEnv "FEDERATION_DOMAIN_1" }}
       - namespaceFed2: {{ requiredEnv "NAMESPACE_2" }}
       - federationDomainFed2: {{ requiredEnv "FEDERATION_DOMAIN_2" }}
-      - kubeVersion: {{ requiredEnv "KUBERNETES_VERSION" | quote }}
+      - kubeVersionAbove1_23: {{ semverCompare ">= 1.23-0" (requiredEnv "KUBERNETES_VERSION" | quote) }}
       - imagePullPolicy: Never
       - redisStorageClass: standard
 
@@ -77,14 +78,14 @@ releases:
   - name: '{{ .Values.namespace }}-ingress-c'
     namespace: '{{ .Values.namespace }}'
     chart: '../.local/charts/ingress-nginx-controller'
-    condition: {{ semverCompare ">= 1.23-0" .Values.kubeVersion }}
+    condition: kubeVersionAbove1_23
     values:
       - './helm_vars/ingress-nginx-controller/values.yaml.gotmpl'
 
   - name: '{{ .Values.namespace }}-ingress-c-2'
     namespace: '{{ .Values.namespaceFed2 }}'
     chart: '../.local/charts/ingress-nginx-controller'
-    condition: {{ semverCompare ">= 1.23-0" .Values.kubeVersion }}
+    condition: kubeVersionAbove1_23
     values:
       - './helm_vars/ingress-nginx-controller/values.yaml.gotmpl'
 
@@ -92,7 +93,7 @@ releases:
   - name: '{{ .Values.namespace }}-ingress-co'
     namespace: '{{ .Values.namespace }}'
     chart: '../.local/charts/nginx-ingress-controller'
-    condition: {{ semverCompare "<= 1.20-0" .Values.kubeVersion }}
+    condition: kubeVersionBelow1_21
     values:
       - './helm_vars/nginx-ingress-controller/values.yaml'
 
@@ -100,7 +101,7 @@ releases:
   - name: '{{ .Values.namespace }}-ingress-co-2'
     namespace: '{{ .Values.namespaceFed2 }}'
     chart: '../.local/charts/nginx-ingress-controller'
-    condition: {{ semverCompare "<= 1.20-0" .Values.kubeVersion }}
+    condition: kubeVersionBelow1_21
     values:
       - './helm_vars/nginx-ingress-controller/values.yaml'
 
diff --git a/nix/overlay.nix b/nix/overlay.nix
index 767abcaa20..d14b111e09 100644
--- a/nix/overlay.nix
+++ b/nix/overlay.nix
@@ -103,21 +103,6 @@ self: super: {
     inherit (super) stdenv fetchurl;
   };
 
-  kubectl = staticBinaryInTarball {
-    pname = "kubectl";
-    version = "1.19.8";
-
-    darwinAmd64Url = "https://dl.k8s.io/v1.19.8/kubernetes-client-darwin-amd64.tar.gz";
-    darwinAmd64Sha256 = "23b847bb8b545c748e9078e7660c654eef74d15ccab8696d294f3d6c619c788e";
-
-    linuxAmd64Url = "https://dl.k8s.io/v1.19.8/kubernetes-client-linux-amd64.tar.gz";
-    linuxAmd64Sha256 = "8388ff8b5c676bdbb8fe07ef7077de937b0bf60154f302df5f248f38f95122aa";
-
-    binPath = "client/bin/kubectl";
-
-    inherit (super) stdenv fetchurl;
-  };
-
   kind = staticBinary {
     pname = "kind";
     version = "0.11.0";

From 4501a33c4b19fba0abadf6ffce252f3bb638b28d Mon Sep 17 00:00:00 2001
From: jschaul <jschaul@users.noreply.github.com>
Date: Thu, 16 Mar 2023 15:36:44 +0100
Subject: [PATCH 20/40] do conditional logic inside bash, not helmfile (as that
 doesn't work it seems)

---
 hack/bin/integration-setup-federation.sh      |  7 ++++-
 hack/bin/integration-setup.sh                 |  7 ++++-
 .../{values.yaml => values.yaml.gotmpl}       |  0
 hack/helmfile.yaml                            | 31 ++++---------------
 4 files changed, 18 insertions(+), 27 deletions(-)
 rename hack/helm_vars/nginx-ingress-controller/{values.yaml => values.yaml.gotmpl} (100%)

diff --git a/hack/bin/integration-setup-federation.sh b/hack/bin/integration-setup-federation.sh
index 6a3a4f7d3a..89e58538ac 100755
--- a/hack/bin/integration-setup-federation.sh
+++ b/hack/bin/integration-setup-federation.sh
@@ -30,7 +30,12 @@ echo "Generating self-signed certificates..."
 KUBERNETES_VERSION_MAJOR="$(kubectl version -o json | jq -r .serverVersion.major)"
 KUBERNETES_VERSION_MINOR="$(kubectl version -o json | jq -r .serverVersion.minor)"
 export KUBERNETES_VERSION="$KUBERNETES_VERSION_MAJOR.$KUBERNETES_VERSION_MINOR"
-echo "kubeVersion: $KUBERNETES_VERSION"
+if (( KUBERNETES_VERSION_MINOR >= 23 )); then
+    export INGRESS_CHART="ingress-nginx-controller"
+else
+    export INGRESS_CHART="nginx-ingress-controller"
+fi
+echo "kubeVersion: $KUBERNETES_VERSION and ingress controller=$INGRESS_CHART"
 export NAMESPACE_1="$NAMESPACE"
 export FEDERATION_DOMAIN_BASE="$NAMESPACE_1.svc.cluster.local"
 export FEDERATION_DOMAIN_1="federation-test-helper.$FEDERATION_DOMAIN_BASE"
diff --git a/hack/bin/integration-setup.sh b/hack/bin/integration-setup.sh
index e48328795b..5b93fc4906 100755
--- a/hack/bin/integration-setup.sh
+++ b/hack/bin/integration-setup.sh
@@ -21,7 +21,12 @@ printf '%s\n' "${charts[@]}" | parallel -P "${HELM_PARALLELISM}" "$DIR/update.sh
 KUBERNETES_VERSION_MAJOR="$(kubectl version -o json | jq -r .serverVersion.major)"
 KUBERNETES_VERSION_MINOR="$(kubectl version -o json | jq -r .serverVersion.minor)"
 export KUBERNETES_VERSION="$KUBERNETES_VERSION_MAJOR.$KUBERNETES_VERSION_MINOR"
-echo "kubeVersion: $KUBERNETES_VERSION"
+if (( KUBERNETES_VERSION_MINOR >= 23 )); then
+    export INGRESS_CHART="ingress-nginx-controller"
+else
+    export INGRESS_CHART="nginx-ingress-controller"
+fi
+echo "kubeVersion: $KUBERNETES_VERSION and ingress controller=$INGRESS_CHART"
 echo "Generating self-signed certificates..."
 export FEDERATION_DOMAIN_BASE="$NAMESPACE.svc.cluster.local"
 export FEDERATION_DOMAIN="federation-test-helper.$FEDERATION_DOMAIN_BASE"
diff --git a/hack/helm_vars/nginx-ingress-controller/values.yaml b/hack/helm_vars/nginx-ingress-controller/values.yaml.gotmpl
similarity index 100%
rename from hack/helm_vars/nginx-ingress-controller/values.yaml
rename to hack/helm_vars/nginx-ingress-controller/values.yaml.gotmpl
diff --git a/hack/helmfile.yaml b/hack/helmfile.yaml
index 1fd1fddd8c..d7dc2d95bc 100644
--- a/hack/helmfile.yaml
+++ b/hack/helmfile.yaml
@@ -18,8 +18,7 @@ environments:
       - federationDomain: {{ requiredEnv "FEDERATION_DOMAIN_1" }}
       - namespaceFed2: {{ requiredEnv "NAMESPACE_2" }}
       - federationDomainFed2: {{ requiredEnv "FEDERATION_DOMAIN_2" }}
-      - kubeVersionAbove1_23: {{ semverCompare ">= 1.23-0" (requiredEnv "KUBERNETES_VERSION" | quote) }}
-      - kubeVersionBelow1_21: {{ semverCompare "< 1.21-0" (requiredEnv "KUBERNETES_VERSION" | quote) }}
+      - ingressChart: {{ requiredEnv "INGRESS_CHART" }}
       - imagePullPolicy: Always
       - redisStorageClass: hcloud-volumes
   kind:
@@ -28,7 +27,7 @@ environments:
       - federationDomain: {{ requiredEnv "FEDERATION_DOMAIN_1" }}
       - namespaceFed2: {{ requiredEnv "NAMESPACE_2" }}
       - federationDomainFed2: {{ requiredEnv "FEDERATION_DOMAIN_2" }}
-      - kubeVersionAbove1_23: {{ semverCompare ">= 1.23-0" (requiredEnv "KUBERNETES_VERSION" | quote) }}
+      - ingressChart: {{ requiredEnv "INGRESS_CHART" }}
       - imagePullPolicy: Never
       - redisStorageClass: standard
 
@@ -77,33 +76,15 @@ releases:
 
   - name: '{{ .Values.namespace }}-ingress-c'
     namespace: '{{ .Values.namespace }}'
-    chart: '../.local/charts/ingress-nginx-controller'
-    condition: kubeVersionAbove1_23
+    chart: '../.local/charts/{{ .Values.ingressChart }}'
     values:
-      - './helm_vars/ingress-nginx-controller/values.yaml.gotmpl'
+      - './helm_vars/{{ .Values.ingressChart }}/values.yaml.gotmpl'
 
   - name: '{{ .Values.namespace }}-ingress-c-2'
     namespace: '{{ .Values.namespaceFed2 }}'
-    chart: '../.local/charts/ingress-nginx-controller'
-    condition: kubeVersionAbove1_23
+    chart: '../.local/charts/{{ .Values.ingressChart }}'
     values:
-      - './helm_vars/ingress-nginx-controller/values.yaml.gotmpl'
-
-  # on old clusters, install the old controller chart
-  - name: '{{ .Values.namespace }}-ingress-co'
-    namespace: '{{ .Values.namespace }}'
-    chart: '../.local/charts/nginx-ingress-controller'
-    condition: kubeVersionBelow1_21
-    values:
-      - './helm_vars/nginx-ingress-controller/values.yaml'
-
-  # on old clusters, install the old controller chart
-  - name: '{{ .Values.namespace }}-ingress-co-2'
-    namespace: '{{ .Values.namespaceFed2 }}'
-    chart: '../.local/charts/nginx-ingress-controller'
-    condition: kubeVersionBelow1_21
-    values:
-      - './helm_vars/nginx-ingress-controller/values.yaml'
+      - './helm_vars/{{ .Values.ingressChart }}/values.yaml.gotmpl'
 
   - name: '{{ .Values.namespace }}-ingresses'
     namespace: '{{ .Values.namespace }}'

From 8ed166a5883118e547f3eec28dd235304cf39eed Mon Sep 17 00:00:00 2001
From: jschaul <jschaul@users.noreply.github.com>
Date: Thu, 16 Mar 2023 15:41:29 +0100
Subject: [PATCH 21/40] Add oidc login to work with kubernetes clusters beind
 oidc

---
 nix/wire-server.nix | 1 +
 1 file changed, 1 insertion(+)

diff --git a/nix/wire-server.nix b/nix/wire-server.nix
index b1883e0405..0d54fabbbf 100644
--- a/nix/wire-server.nix
+++ b/nix/wire-server.nix
@@ -306,6 +306,7 @@ let
     (hlib.justStaticExecutables pkgs.haskellPackages.apply-refact)
     pkgs.jq
     pkgs.kubectl
+    pkgs.kubelogin-oidc
     pkgs.nixpkgs-fmt
     pkgs.ormolu
     pkgs.shellcheck

From a2cb5043fb35a3f8b46cd66a7575ed65bdf40ad8 Mon Sep 17 00:00:00 2001
From: jschaul <jschaul@users.noreply.github.com>
Date: Thu, 16 Mar 2023 17:00:30 +0100
Subject: [PATCH 22/40] also set INGRESS_CHART in teardown script

---
 hack/bin/integration-teardown-federation.sh | 7 +++++++
 hack/bin/integration-teardown.sh            | 6 ++++++
 2 files changed, 13 insertions(+)

diff --git a/hack/bin/integration-teardown-federation.sh b/hack/bin/integration-teardown-federation.sh
index 91897a6059..348724e5f3 100755
--- a/hack/bin/integration-teardown-federation.sh
+++ b/hack/bin/integration-teardown-federation.sh
@@ -12,6 +12,13 @@ export NAMESPACE_2="$NAMESPACE-fed2"
 export FEDERATION_DOMAIN_1="."
 export FEDERATION_DOMAIN_2="."
 
+KUBERNETES_VERSION_MINOR="$(kubectl version -o json | jq -r .serverVersion.minor)"
+if (( KUBERNETES_VERSION_MINOR >= 23 )); then
+    export INGRESS_CHART="ingress-nginx-controller"
+else
+    export INGRESS_CHART="nginx-ingress-controller"
+fi
+
 . "$DIR/helm_overrides.sh"
 helmfile --file "${TOP_LEVEL}/hack/helmfile.yaml" destroy
 
diff --git a/hack/bin/integration-teardown.sh b/hack/bin/integration-teardown.sh
index cd82194c2b..6de5bdecec 100755
--- a/hack/bin/integration-teardown.sh
+++ b/hack/bin/integration-teardown.sh
@@ -6,6 +6,12 @@ TOP_LEVEL="$DIR/../.."
 NAMESPACE=${NAMESPACE:-test-integration}
 # doesn't matter for destruction but needs to be set
 export FEDERATION_DOMAIN="."
+KUBERNETES_VERSION_MINOR="$(kubectl version -o json | jq -r .serverVersion.minor)"
+if (( KUBERNETES_VERSION_MINOR >= 23 )); then
+    export INGRESS_CHART="ingress-nginx-controller"
+else
+    export INGRESS_CHART="nginx-ingress-controller"
+fi
 
 set -ex
 

From abe1c185efb042da7b7338a2f0e6a10b9d34e76d Mon Sep 17 00:00:00 2001
From: jschaul <jschaul@users.noreply.github.com>
Date: Thu, 16 Mar 2023 17:45:24 +0100
Subject: [PATCH 23/40] update outdated instructions from old readme

---
 charts/nginx-ingress-services/README.md | 14 +++-----------
 1 file changed, 3 insertions(+), 11 deletions(-)

diff --git a/charts/nginx-ingress-services/README.md b/charts/nginx-ingress-services/README.md
index 50eb736fa8..2dadd73b38 100644
--- a/charts/nginx-ingress-services/README.md
+++ b/charts/nginx-ingress-services/README.md
@@ -1,5 +1,5 @@
 This helm chart is a helper to set up needed services, ingresses and (likely) secrets to access your cluster.
-It will _NOT_ deploy an ingress controller! Ensure you already have one on your cluster - or have a look at our [nginx-ingress-controller](../nginx-ingress-controller/README.md)
+It will _NOT_ deploy an ingress controller! Ensure you already have one on your cluster - or have a look at our [ingress-nginx-controller](../ingress-nginx-controller/README.md)
 
 If tls.enabled == true, then you need to supply 2 variables, `tlsWildcardCert` and `tlsWildcardKey` that could either be supplied as plain text in the form of a `-f path/to/secrets.yaml`, like this:
 
@@ -28,16 +28,14 @@ Q: My ingress keeps serving "Kubernetes Ingress Controller Fake Certificate"!!
 
 A: Ensure that your certificate is _valid_ and has _not expired_; trying to serve expired certificates will silently fail and the nginx ingress will simply fallback to the default certificate.
 
-
 ## About cert-manager
 
 ### Prerequisites
 
-* `cert-manager` and its CRDs have to be installed upfront, 
+* `cert-manager` and its CRDs have to be installed upfront,
    e.g. `helm upgrade --install -n cert-manager-ns --set 'installCRDs=true' cert-manager jetstack/cert-manager`,
    because upstream decided that this is the way (https://github.com/jetstack/cert-manager/pull/2964)
 
-
 ### What does this chart do?
 
 * define `Ingress` for various services and their corresponding FQDNS
@@ -45,10 +43,4 @@ A: Ensure that your certificate is _valid_ and has _not expired_; trying to serv
   *cert-manager* take care of this
 * [optional] configure an *Issuer* to issue ACME HTTP01 certificates provided by Letsencrypt
 * [optional] define a *Certificate* representation that causes *cert-manager* to issue a
-  certificate that is then used by `Ingress` 
-
-
-### Todo when introducing support for K8s >= 1.15 
-
-* the `apiVersion` of all resources based on cert-manager's CRDs, namely `./templates/issuer.yaml` and 
-  `./templates/certificate.yaml`, has to be changed to `cert-manager.io/v1alpha3`
+  certificate that is then used by `Ingress`

From cba54b0aef7b924152dc516d3197f7c71a44580a Mon Sep 17 00:00:00 2001
From: jschaul <jschaul@users.noreply.github.com>
Date: Thu, 16 Mar 2023 17:46:24 +0100
Subject: [PATCH 24/40] allow overriding ingress class; override it for
 integration tests

---
 charts/nginx-ingress-services/templates/ingress.yaml           | 2 +-
 charts/nginx-ingress-services/templates/ingress_federator.yaml | 2 +-
 charts/nginx-ingress-services/values.yaml                      | 3 ++-
 .../nginx-ingress-services/{values.yaml => values.yaml.gotmpl} | 1 +
 4 files changed, 5 insertions(+), 3 deletions(-)
 rename hack/helm_vars/nginx-ingress-services/{values.yaml => values.yaml.gotmpl} (92%)

diff --git a/charts/nginx-ingress-services/templates/ingress.yaml b/charts/nginx-ingress-services/templates/ingress.yaml
index 0314ee9513..6b19cdd090 100644
--- a/charts/nginx-ingress-services/templates/ingress.yaml
+++ b/charts/nginx-ingress-services/templates/ingress.yaml
@@ -5,7 +5,7 @@ kind: Ingress
 metadata:
   name: nginx-ingress
   annotations:
-    kubernetes.io/ingress.class: "nginx"
+    kubernetes.io/ingress.class: "{{ .Values.config.ingressClass }}"
 spec:
   # This assumes you have created the given cert (see secret.yaml)
   # https://github.com/kubernetes/ingress-nginx/blob/master/docs/examples/PREREQUISITES.md#tls-certificates
diff --git a/charts/nginx-ingress-services/templates/ingress_federator.yaml b/charts/nginx-ingress-services/templates/ingress_federator.yaml
index 8d52ff9a33..0318d9ce5b 100644
--- a/charts/nginx-ingress-services/templates/ingress_federator.yaml
+++ b/charts/nginx-ingress-services/templates/ingress_federator.yaml
@@ -8,7 +8,7 @@ kind: Ingress
 metadata:
   name: federator-ingress
   annotations:
-    kubernetes.io/ingress.class: "nginx"
+    kubernetes.io/ingress.class: "{{ .Values.config.ingressClass }}"
     nginx.ingress.kubernetes.io/ssl-redirect: "true"
     nginx.ingress.kubernetes.io/backend-protocol: "HTTP"
     nginx.ingress.kubernetes.io/auth-tls-verify-client: "on"
diff --git a/charts/nginx-ingress-services/values.yaml b/charts/nginx-ingress-services/values.yaml
index b76a61390d..259d80540f 100644
--- a/charts/nginx-ingress-services/values.yaml
+++ b/charts/nginx-ingress-services/values.yaml
@@ -91,8 +91,9 @@ service:
   accountPages:
     externalPort: 8080
 
+config:
+  ingressClass: "nginx"
 # You will need to supply some DNS names, namely
-# config:
 #   dns:
 #     https: nginz-https.<domain>
 #     ssl: nginz-ssl.<domain> # For websockets
diff --git a/hack/helm_vars/nginx-ingress-services/values.yaml b/hack/helm_vars/nginx-ingress-services/values.yaml.gotmpl
similarity index 92%
rename from hack/helm_vars/nginx-ingress-services/values.yaml
rename to hack/helm_vars/nginx-ingress-services/values.yaml.gotmpl
index 76aa0657e8..fd80c5ca43 100644
--- a/hack/helm_vars/nginx-ingress-services/values.yaml
+++ b/hack/helm_vars/nginx-ingress-services/values.yaml.gotmpl
@@ -9,6 +9,7 @@ tls:
   useCertManager: false
 
 config:
+  ingressClass: "nginx-{{ .Release.Namespace }}"
   dns:
     https: nginz-https.integration.example.com
     ssl: nginz-ssl.integration.example.com

From 1f726963998a5cd27af400df5684e6b472adc980 Mon Sep 17 00:00:00 2001
From: jschaul <jschaul@users.noreply.github.com>
Date: Thu, 16 Mar 2023 18:27:27 +0100
Subject: [PATCH 25/40] ...

---
 hack/helmfile.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/hack/helmfile.yaml b/hack/helmfile.yaml
index d7dc2d95bc..1a8806db1a 100644
--- a/hack/helmfile.yaml
+++ b/hack/helmfile.yaml
@@ -90,7 +90,7 @@ releases:
     namespace: '{{ .Values.namespace }}'
     chart: '../.local/charts/nginx-ingress-services'
     values:
-      - './helm_vars/nginx-ingress-services/values.yaml'
+      - './helm_vars/nginx-ingress-services/values.yaml.gotmpl'
       - './helm_vars/nginx-ingress-services/certificates-namespace1.yaml'
     set:
       # Federation domain is also the SRV record created by the
@@ -103,7 +103,7 @@ releases:
     namespace: '{{ .Values.namespaceFed2 }}'
     chart: '../.local/charts/nginx-ingress-services'
     values:
-      - './helm_vars/nginx-ingress-services/values.yaml'
+      - './helm_vars/nginx-ingress-services/values.yaml.gotmpl'
       - './helm_vars/nginx-ingress-services/certificates-namespace2.yaml'
     set:
       # Federation domain is also the SRV record created by the

From 17f5190a3d27e3eb866bb13caa8ba564b0ca49a0 Mon Sep 17 00:00:00 2001
From: jschaul <jschaul@users.noreply.github.com>
Date: Tue, 21 Mar 2023 12:34:47 +0100
Subject: [PATCH 26/40] update helmfile to latest

---
 nix/overlay.nix | 13 -------------
 1 file changed, 13 deletions(-)

diff --git a/nix/overlay.nix b/nix/overlay.nix
index d14b111e09..2f02487a65 100644
--- a/nix/overlay.nix
+++ b/nix/overlay.nix
@@ -90,19 +90,6 @@ self: super: {
 
   helm = super.callPackage ./pkgs/helm { };
 
-  helmfile = staticBinary {
-    pname = "helmfile";
-    version = "0.141.0";
-
-    darwinAmd64Url = "https://github.com/roboll/helmfile/releases/download/v0.141.0/helmfile_darwin_amd64";
-    darwinAmd64Sha256 = "0szfd3vy6fzd5657079hz5vii86f9xkg3bdzp3g4knkcw5x1kpxy";
-
-    linuxAmd64Url = "https://github.com/roboll/helmfile/releases/download/v0.141.0/helmfile_linux_amd64";
-    linuxAmd64Sha256 = "0f5d9w3qjvwip4qn79hsigwp8nbjpj58p289hww503j43wjyxx8r";
-
-    inherit (super) stdenv fetchurl;
-  };
-
   kind = staticBinary {
     pname = "kind";
     version = "0.11.0";

From e0a24de31f585730d02cb37df6d1dc84cff75e8f Mon Sep 17 00:00:00 2001
From: jschaul <jschaul@users.noreply.github.com>
Date: Tue, 21 Mar 2023 12:36:36 +0100
Subject: [PATCH 27/40] remove patched helm binary; use default from nixpkgs

---
 nix/overlay.nix           |  2 --
 nix/pkgs/helm/default.nix | 52 ---------------------------------------
 nix/wire-server.nix       |  2 +-
 3 files changed, 1 insertion(+), 55 deletions(-)
 delete mode 100644 nix/pkgs/helm/default.nix

diff --git a/nix/overlay.nix b/nix/overlay.nix
index 2f02487a65..f5f8853331 100644
--- a/nix/overlay.nix
+++ b/nix/overlay.nix
@@ -88,8 +88,6 @@ self: super: {
     inherit (super) stdenv fetchurl;
   };
 
-  helm = super.callPackage ./pkgs/helm { };
-
   kind = staticBinary {
     pname = "kind";
     version = "0.11.0";
diff --git a/nix/pkgs/helm/default.nix b/nix/pkgs/helm/default.nix
deleted file mode 100644
index 8b68403913..0000000000
--- a/nix/pkgs/helm/default.nix
+++ /dev/null
@@ -1,52 +0,0 @@
-# Copied from nixpkgs and modified because it seems too complicated to override
-# buildGoModule packages.
-{ lib, stdenv, buildGoModule, fetchFromGitHub, installShellFiles }:
-
-buildGoModule rec {
-  pname = "kubernetes-helm";
-  version = "3.11.0-patched";
-
-  src = fetchFromGitHub {
-    owner = "wireapp";
-    repo = "helm";
-    rev = "949de3195be5b3d21ed707da18ee3bcb2a9a2af8";
-    sha256 = "sha256-alyR6+gm7WEvFfJxHl9a0jpC3+457Kg6aRHcidA0RZg=";
-  };
-  vendorSha256 = "sha256-LRMDrBSl5EGQqQt5FUU4JJHqdwfYt5qsVpe76jUQBVI=";
-
-  subPackages = [ "cmd/helm" ];
-  ldflags = [
-    "-w"
-    "-s"
-    "-X helm.sh/helm/v3/internal/version.version=v${version}"
-    "-X helm.sh/helm/v3/internal/version.gitCommit=${src.rev}"
-  ];
-
-  preCheck = ''
-    # skipping version tests because they require dot git directory
-    substituteInPlace cmd/helm/version_test.go \
-      --replace "TestVersion" "SkipVersion"
-  '' + lib.optionalString stdenv.isLinux ''
-    # skipping plugin tests on linux
-    substituteInPlace cmd/helm/plugin_test.go \
-      --replace "TestPluginDynamicCompletion" "SkipPluginDynamicCompletion" \
-      --replace "TestLoadPlugins" "SkipLoadPlugins"
-    substituteInPlace cmd/helm/helm_test.go \
-      --replace "TestPluginExitCode" "SkipPluginExitCode"
-  '';
-
-  nativeBuildInputs = [ installShellFiles ];
-  postInstall = ''
-    $out/bin/helm completion bash > helm.bash
-    $out/bin/helm completion zsh > helm.zsh
-    installShellCompletion helm.{bash,zsh}
-  '';
-
-  meta = with lib; {
-    homepage = "https://github.com/kubernetes/helm";
-    description = "A package manager for kubernetes";
-    mainProgram = "helm";
-    license = licenses.asl20;
-    maintainers = with maintainers; [ rlupton20 edude03 saschagrunert Frostman Chili-Man techknowlogick ];
-  };
-}
diff --git a/nix/wire-server.nix b/nix/wire-server.nix
index 0d54fabbbf..c41815483a 100644
--- a/nix/wire-server.nix
+++ b/nix/wire-server.nix
@@ -300,7 +300,7 @@ let
     pkgs.gnused
     pkgs.parallel
     pkgs.ripgrep
-    pkgs.helm
+    pkgs.kubernetes-helm
     pkgs.helmfile
     pkgs.hlint
     (hlib.justStaticExecutables pkgs.haskellPackages.apply-refact)

From 3838742a080718fbcbdf9fd3aea60b46ee57cc04 Mon Sep 17 00:00:00 2001
From: jschaul <jschaul@users.noreply.github.com>
Date: Tue, 21 Mar 2023 12:52:37 +0100
Subject: [PATCH 28/40] helmfile: needs

---
 hack/helmfile.yaml | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/hack/helmfile.yaml b/hack/helmfile.yaml
index 1a8806db1a..f9e608107e 100644
--- a/hack/helmfile.yaml
+++ b/hack/helmfile.yaml
@@ -74,19 +74,19 @@ releases:
     values:
       - './helm_vars/redis-cluster/values.yaml.gotmpl'
 
-  - name: '{{ .Values.namespace }}-ingress-c'
+  - name: '{{ .Values.namespace }}-ic'
     namespace: '{{ .Values.namespace }}'
     chart: '../.local/charts/{{ .Values.ingressChart }}'
     values:
       - './helm_vars/{{ .Values.ingressChart }}/values.yaml.gotmpl'
 
-  - name: '{{ .Values.namespace }}-ingress-c-2'
+  - name: '{{ .Values.namespace }}-ic2'
     namespace: '{{ .Values.namespaceFed2 }}'
     chart: '../.local/charts/{{ .Values.ingressChart }}'
     values:
       - './helm_vars/{{ .Values.ingressChart }}/values.yaml.gotmpl'
 
-  - name: '{{ .Values.namespace }}-ingresses'
+  - name: '{{ .Values.namespace }}-i'
     namespace: '{{ .Values.namespace }}'
     chart: '../.local/charts/nginx-ingress-services'
     values:
@@ -98,8 +98,10 @@ releases:
       # differ, so we don't make any silly assumptions in the code.
       - name: config.dns.federator
         value: {{ .Values.federationDomain }}
+    needs:
+      - '{{ .Values.namespace }}-ic'
 
-  - name: '{{ .Values.namespace }}-ingresses-2'
+  - name: '{{ .Values.namespace }}-i2'
     namespace: '{{ .Values.namespaceFed2 }}'
     chart: '../.local/charts/nginx-ingress-services'
     values:
@@ -111,6 +113,8 @@ releases:
       # differ, so we don't make any silly assumptions in the code.
       - name: config.dns.federator
         value: {{ .Values.federationDomainFed2 }}
+    needs:
+      - '{{ .Values.namespace }}-ic2'
 
   # Note that wire-server depends on databases-ephemeral being up; and in some
   # cases on nginx-ingress also being up. If installing helm charts in a

From f6b8df6ce4f2592a792a325574e6d69e62041dd9 Mon Sep 17 00:00:00 2001
From: jschaul <jschaul@users.noreply.github.com>
Date: Tue, 21 Mar 2023 13:02:02 +0100
Subject: [PATCH 29/40] don't watch all ingresses, but only those in the right
 class

---
 charts/ingress-nginx-controller/values.yaml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/charts/ingress-nginx-controller/values.yaml b/charts/ingress-nginx-controller/values.yaml
index 1b796d54a6..1402388bfe 100644
--- a/charts/ingress-nginx-controller/values.yaml
+++ b/charts/ingress-nginx-controller/values.yaml
@@ -6,7 +6,6 @@
 # See https://github.com/kubernetes/ingress-nginx/blob/main/charts/ingress-nginx/values.yaml for all possible values to override.
 ingress-nginx:
   controller:
-    watchIngressWithoutClass: true
     enableTopologyAwareRouting: true
     # -- Use a `DaemonSet` or `Deployment`
     kind: DaemonSet

From c92ee4251f886567d7ab27e6f479684156055bcb Mon Sep 17 00:00:00 2001
From: jschaul <jschaul@users.noreply.github.com>
Date: Tue, 21 Mar 2023 13:13:21 +0100
Subject: [PATCH 30/40] override ingressClass in CI consistently

---
 charts/nginx-ingress-controller/values.yaml                | 1 +
 hack/helm_vars/nginx-ingress-controller/values.yaml.gotmpl | 4 +++-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/charts/nginx-ingress-controller/values.yaml b/charts/nginx-ingress-controller/values.yaml
index 6063fa239a..7d1bc21c93 100644
--- a/charts/nginx-ingress-controller/values.yaml
+++ b/charts/nginx-ingress-controller/values.yaml
@@ -43,6 +43,7 @@ nginx-ingress:
 # downsides of this setting
 #   https://www.asykim.com/blog/deep-dive-into-kubernetes-external-traffic-policies
     kind: DaemonSet
+    ingressClass: nginx
     # By default, each node will now be configured to accept ingress traffic. You should add
     # all the nodes to your external load balancer, or add them to DNS records.
     #
diff --git a/hack/helm_vars/nginx-ingress-controller/values.yaml.gotmpl b/hack/helm_vars/nginx-ingress-controller/values.yaml.gotmpl
index 53b1c57506..10fd76e22b 100644
--- a/hack/helm_vars/nginx-ingress-controller/values.yaml.gotmpl
+++ b/hack/helm_vars/nginx-ingress-controller/values.yaml.gotmpl
@@ -1,6 +1,8 @@
 nginx-ingress:
   controller:
-    kind: DaemonSet
+    kind: Deployment
+    replicaCount: 1
+    ingressClass: "nginx-{{ .Release.Namespace }}"
     service:
       type: NodePort
       externalTrafficPolicy: Local

From db3e342ed3265d8032388c36916a7f4625a44960 Mon Sep 17 00:00:00 2001
From: jschaul <jschaul@users.noreply.github.com>
Date: Tue, 21 Mar 2023 13:23:24 +0100
Subject: [PATCH 31/40] disable validation webhooks in CI

---
 hack/helm_vars/ingress-nginx-controller/values.yaml.gotmpl | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/hack/helm_vars/ingress-nginx-controller/values.yaml.gotmpl b/hack/helm_vars/ingress-nginx-controller/values.yaml.gotmpl
index eea4a26a83..d3c24971e6 100644
--- a/hack/helm_vars/ingress-nginx-controller/values.yaml.gotmpl
+++ b/hack/helm_vars/ingress-nginx-controller/values.yaml.gotmpl
@@ -12,3 +12,8 @@ ingress-nginx:
         # choose a random free port
         https: null
         http: null
+      # in CI, do not use ValidatingWebhooks, as these, if not properly cleaned up
+      # (i.e. the ingress controller was deleted in another namespace but the webhook remains)
+      # prevent new kind:Ingress resources to be created in the cluster.
+      admissionWebhooks:
+        enabled: false

From f8196c09e6e41bb1419a8690716895162fa7f545 Mon Sep 17 00:00:00 2001
From: jschaul <jschaul@users.noreply.github.com>
Date: Tue, 21 Mar 2023 13:28:47 +0100
Subject: [PATCH 32/40] also adjust helmfile-single

---
 hack/helmfile-single.yaml | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/hack/helmfile-single.yaml b/hack/helmfile-single.yaml
index 3a770ee146..bb9f4f5b2f 100644
--- a/hack/helmfile-single.yaml
+++ b/hack/helmfile-single.yaml
@@ -13,7 +13,9 @@ environments:
     values:
       - namespace: {{ requiredEnv "NAMESPACE" }}
       - federationDomain: {{ requiredEnv "FEDERATION_DOMAIN" }}
+      - ingressChart: {{ requiredEnv "INGRESS_CHART" }}
       - imagePullPolicy: Always
+      - redisStorageClass: hcloud-volumes
 
 repositories:
   - name: stable
@@ -39,17 +41,17 @@ releases:
     values:
       - './helm_vars/redis-cluster/values.yaml.gotmpl'
 
-  - name: '{{ .Values.namespace }}-nginx-ingress-controller'
+  - name: '{{ .Values.namespace }}-ic'
     namespace: '{{ .Values.namespace }}'
-    chart: '../.local/charts/nginx-ingress-controller'
+    chart: '../.local/charts/{{ .Values.ingressChart }}'
     values:
-      - './helm_vars/nginx-ingress-controller/values.yaml'
+      - './helm_vars/{{ .Values.ingressChart }}/values.yaml.gotmpl'
 
-  - name: '{{ .Values.namespace }}-nginx-ingress-services'
+  - name: '{{ .Values.namespace }}-i'
     namespace: '{{ .Values.namespace }}'
     chart: '../.local/charts/nginx-ingress-services'
     values:
-      - './helm_vars/nginx-ingress-services/values.yaml'
+      - './helm_vars/nginx-ingress-services/values.yaml.gotmpl'
       - './helm_vars/nginx-ingress-services/certificates-namespace1.yaml'
     set:
       # Federation domain is also the SRV record created by the
@@ -57,6 +59,8 @@ releases:
       # differ, so we don't make any silly assumptions in the code.
       - name: config.dns.federator
         value: {{ .Values.federationDomain }}
+    needs:
+      - '{{ .Values.namespace }}-ic'
 
   # Note that wire-server depends on databases-ephemeral being up; and in some
   # cases on nginx-ingress also being up. If installing helm charts in a

From 049ace76c1c3efda3315c59646f02306fdaf3ef2 Mon Sep 17 00:00:00 2001
From: jschaul <jschaul@users.noreply.github.com>
Date: Tue, 21 Mar 2023 14:17:37 +0100
Subject: [PATCH 33/40] adjust federation-test-helper service to match new
 ingress-controller

---
 charts/nginx-ingress-services/templates/_helpers.tpl        | 4 ++++
 .../templates/federation-test-helper.yaml                   | 6 ++++++
 2 files changed, 10 insertions(+)

diff --git a/charts/nginx-ingress-services/templates/_helpers.tpl b/charts/nginx-ingress-services/templates/_helpers.tpl
index 32f4467617..7b0faa4711 100644
--- a/charts/nginx-ingress-services/templates/_helpers.tpl
+++ b/charts/nginx-ingress-services/templates/_helpers.tpl
@@ -86,3 +86,7 @@ Returns the Letsencrypt API server URL based on whether testMode is enabled or d
 {{- define "ingress.supportsPathType" -}}
   {{- or (eq (include "ingress.isStable" .) "true") (and (eq (include "ingress.apiVersion" .) "networking.k8s.io/v1beta1") (semverCompare ">= 1.18-0" (include "kubeVersion" .))) -}}
 {{- end -}}
+
+{{- define "integrationTestHelperNewLabels" -}}
+  {{- (semverCompare ">= 1.23-0" (include "kubeVersion" .)) -}}
+{{- end -}}
diff --git a/charts/nginx-ingress-services/templates/federation-test-helper.yaml b/charts/nginx-ingress-services/templates/federation-test-helper.yaml
index 4c06577052..0fb621e34d 100644
--- a/charts/nginx-ingress-services/templates/federation-test-helper.yaml
+++ b/charts/nginx-ingress-services/templates/federation-test-helper.yaml
@@ -1,3 +1,4 @@
+{{- $newLabels := eq (include "integrationTestHelperNewLabels" .) "true" -}}
 # Assumes that the controller is deployed in the same namespace. Only used for
 # enabling discovery by creating SRV records while running integration tests.
 {{- if (and .Values.federator.enabled .Values.federator.integrationTestHelper) }}
@@ -13,7 +14,12 @@ spec:
     protocol: TCP
     targetPort: https
   selector:
+    {{- if $newLabels }}
+    app.kubernetes.io/component: controller
+    app.kubernetes.io/name: ingress-nginx
+    {{- else }}
     app: nginx-ingress
     component: controller
+    {{- end }}
   type: ClusterIP
 {{- end }}

From 4d463efb25f4db964c4f7a24bf2a0c223e81e40a Mon Sep 17 00:00:00 2001
From: jschaul <jschaul@users.noreply.github.com>
Date: Tue, 21 Mar 2023 14:26:02 +0100
Subject: [PATCH 34/40] adjust changelog

---
 changelog.d/0-release-notes/ingress-controller | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/changelog.d/0-release-notes/ingress-controller b/changelog.d/0-release-notes/ingress-controller
index 0d381e6815..85c52487f6 100644
--- a/changelog.d/0-release-notes/ingress-controller
+++ b/changelog.d/0-release-notes/ingress-controller
@@ -1,5 +1,5 @@
-New 'ingress-nginx-controller' wrapper chart compatible with k8s versions [1.23 - 1.26]. The old one 'nginx-ingress-controller' (compatible only up to k8s 1.19) is now DEPRECATED.
-We advise to upgrade your version of kubernetes in use, and to make use of the new ingress controller chart. Main features:
+New 'ingress-nginx-controller' wrapper chart compatible with kubernetes versions [1.23 - 1.26]. The old one 'nginx-ingress-controller' (compatible only up to k8s 1.19) is now DEPRECATED.
+We advise to upgrade your version of kubernetes in use to 1.23 or higher (we tested on kubernetes version 1.26), and to make use of the new ingress controller chart. Main features:
 - up-to-date nginx version ('1.21.6')
 - TLS 1.3 support (including allowing specifying which cipher suites to use)
 - security fixes
@@ -20,3 +20,16 @@ In case you have custom overrides, you need to modify the directory name and top
 ```
 
 and double-check if all overrides you use are indeed provided under the same name by the upstream chart. See also the default overrides in [the default values.yaml](https://github.com/wireapp/wire-server/blob/develop/charts/ingress-nginx-controller/values.yaml).
+
+In case you use helmfile change your ingress controller like this:
+
+```diff
+# helmfile.yaml
+releases:
+-  - name: 'nginx-ingress-controller'
++  - name: 'ingress-nginx-controller'
+     namespace: 'wire'
+-    chart: 'wire/nginx-ingress-controller'
++    chart: 'wire/ingress-nginx-controller'
+     version: 'CHANGE_ME'
+```

From 3bb1b205f02ce850e1259a740c00076afbd5af02 Mon Sep 17 00:00:00 2001
From: jschaul <jschaul@users.noreply.github.com>
Date: Tue, 21 Mar 2023 14:47:29 +0100
Subject: [PATCH 35/40] also add changelog for internal changes

---
 changelog.d/5-internal/ingress-controller | 5 +++++
 1 file changed, 5 insertions(+)
 create mode 100644 changelog.d/5-internal/ingress-controller

diff --git a/changelog.d/5-internal/ingress-controller b/changelog.d/5-internal/ingress-controller
new file mode 100644
index 0000000000..7d79310d91
--- /dev/null
+++ b/changelog.d/5-internal/ingress-controller
@@ -0,0 +1,5 @@
+- integration tests on CI will use either the old or the new ingress controller; depending on which kubernetes version they run on.
+- upgrade `kubectl` to default from the nixpkgs channel (currently `1.26`) by removing the manual version pin on 1.19
+- upgrade `helmfile` to default from the nixpkgs channel by removing the manual version pin
+- upgrade `helm` to default from the nixpkgs channel by removing the manual version pin
+- add `kubelogin-oidc` so the kubectl in this environment can also talk to kubernetes clusters using OIDC

From d4717145c6947841ef806c80d4d07c60d1640466 Mon Sep 17 00:00:00 2001
From: jschaul <jschaul@users.noreply.github.com>
Date: Wed, 22 Mar 2023 11:46:44 +0100
Subject: [PATCH 36/40] Apply suggestions from code review

Co-authored-by: Sebastian Willenborg <sebastian.willenborg@wire.com>
---
 hack/bin/integration-setup-federation.sh    | 2 +-
 hack/bin/integration-setup.sh               | 2 +-
 hack/bin/integration-teardown-federation.sh | 2 +-
 hack/bin/integration-teardown.sh            | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/hack/bin/integration-setup-federation.sh b/hack/bin/integration-setup-federation.sh
index 89e58538ac..f624dc1167 100755
--- a/hack/bin/integration-setup-federation.sh
+++ b/hack/bin/integration-setup-federation.sh
@@ -30,7 +30,7 @@ echo "Generating self-signed certificates..."
 KUBERNETES_VERSION_MAJOR="$(kubectl version -o json | jq -r .serverVersion.major)"
 KUBERNETES_VERSION_MINOR="$(kubectl version -o json | jq -r .serverVersion.minor)"
 export KUBERNETES_VERSION="$KUBERNETES_VERSION_MAJOR.$KUBERNETES_VERSION_MINOR"
-if (( KUBERNETES_VERSION_MINOR >= 23 )); then
+if (( KUBERNETES_VERSION_MAJOR > 1 || KUBERNETES_VERSION_MAJOR == 1 && KUBERNETES_VERSION_MINOR >= 23 )); then
     export INGRESS_CHART="ingress-nginx-controller"
 else
     export INGRESS_CHART="nginx-ingress-controller"
diff --git a/hack/bin/integration-setup.sh b/hack/bin/integration-setup.sh
index 5b93fc4906..ed6be40c9e 100755
--- a/hack/bin/integration-setup.sh
+++ b/hack/bin/integration-setup.sh
@@ -21,7 +21,7 @@ printf '%s\n' "${charts[@]}" | parallel -P "${HELM_PARALLELISM}" "$DIR/update.sh
 KUBERNETES_VERSION_MAJOR="$(kubectl version -o json | jq -r .serverVersion.major)"
 KUBERNETES_VERSION_MINOR="$(kubectl version -o json | jq -r .serverVersion.minor)"
 export KUBERNETES_VERSION="$KUBERNETES_VERSION_MAJOR.$KUBERNETES_VERSION_MINOR"
-if (( KUBERNETES_VERSION_MINOR >= 23 )); then
+if (( KUBERNETES_VERSION_MAJOR > 1 || KUBERNETES_VERSION_MAJOR == 1 && KUBERNETES_VERSION_MINOR >= 23 )); then
     export INGRESS_CHART="ingress-nginx-controller"
 else
     export INGRESS_CHART="nginx-ingress-controller"
diff --git a/hack/bin/integration-teardown-federation.sh b/hack/bin/integration-teardown-federation.sh
index 348724e5f3..a439ab6219 100755
--- a/hack/bin/integration-teardown-federation.sh
+++ b/hack/bin/integration-teardown-federation.sh
@@ -13,7 +13,7 @@ export FEDERATION_DOMAIN_1="."
 export FEDERATION_DOMAIN_2="."
 
 KUBERNETES_VERSION_MINOR="$(kubectl version -o json | jq -r .serverVersion.minor)"
-if (( KUBERNETES_VERSION_MINOR >= 23 )); then
+if (( KUBERNETES_VERSION_MAJOR > 1 || KUBERNETES_VERSION_MAJOR == 1 && KUBERNETES_VERSION_MINOR >= 23 )); then
     export INGRESS_CHART="ingress-nginx-controller"
 else
     export INGRESS_CHART="nginx-ingress-controller"
diff --git a/hack/bin/integration-teardown.sh b/hack/bin/integration-teardown.sh
index 6de5bdecec..60f1781b53 100755
--- a/hack/bin/integration-teardown.sh
+++ b/hack/bin/integration-teardown.sh
@@ -7,7 +7,7 @@ NAMESPACE=${NAMESPACE:-test-integration}
 # doesn't matter for destruction but needs to be set
 export FEDERATION_DOMAIN="."
 KUBERNETES_VERSION_MINOR="$(kubectl version -o json | jq -r .serverVersion.minor)"
-if (( KUBERNETES_VERSION_MINOR >= 23 )); then
+if (( KUBERNETES_VERSION_MAJOR > 1 || KUBERNETES_VERSION_MAJOR == 1 && KUBERNETES_VERSION_MINOR >= 23 )); then
     export INGRESS_CHART="ingress-nginx-controller"
 else
     export INGRESS_CHART="nginx-ingress-controller"

From 96d270f8f59c0faa1093aad6e3f5d93821571f52 Mon Sep 17 00:00:00 2001
From: jschaul <jschaul@users.noreply.github.com>
Date: Wed, 22 Mar 2023 14:54:20 +0100
Subject: [PATCH 37/40] Add docs; switch defaults to Load Balancer as suggested
 in PR review

---
 charts/ingress-nginx-controller/values.yaml | 47 ++++++----
 docs/src/how-to/install/ingress.md          | 97 +++++++++++++++++++++
 2 files changed, 127 insertions(+), 17 deletions(-)
 create mode 100644 docs/src/how-to/install/ingress.md

diff --git a/charts/ingress-nginx-controller/values.yaml b/charts/ingress-nginx-controller/values.yaml
index 1402388bfe..80eb20ed9e 100644
--- a/charts/ingress-nginx-controller/values.yaml
+++ b/charts/ingress-nginx-controller/values.yaml
@@ -1,30 +1,43 @@
-# the following defaults apply to an on-prem bare-metal setup in the same spirit as the
-# older similarly named wrapper chart 'nginx-ingress-controller' (note the swapped words
-# 'nginx' and 'ingress') We assume no load balancer support and instead expose NodePorts
-# on ports 31773 and 31772, assuming traffic gets to these ports in another way.
+# The following defaults apply to a cloud-like setup (in which you can ask your
+# kubernetes installation to give you a LoadBalancer setup).
 #
-# See https://github.com/kubernetes/ingress-nginx/blob/main/charts/ingress-nginx/values.yaml for all possible values to override.
+# If you are on bare metal and wish an installation similiar in spirit as the
+# older similarly named wrapper chart 'nginx-ingress-controller' (note the
+# swapped words 'nginx' and 'ingress'), where we assume no load balancer support
+# and instead expose NodePorts on ports 31773 and 31772, and where you need to
+# ensure traffic gets to these ports in another way; then please read the
+# documentation on https://docs.wire.com (or search for "ingress")
+#
+# See
+# https://github.com/kubernetes/ingress-nginx/blob/main/charts/ingress-nginx/values.yaml
+# for all possible values to override.
 ingress-nginx:
   controller:
     enableTopologyAwareRouting: true
-    # -- Use a `DaemonSet` or `Deployment`
-    kind: DaemonSet
+    # Use kind: `DaemonSet` (when using NodePort) or `Deployment` (when using
+    # LoadBalancer)
+    kind: Deployment
     service:
-      type: NodePort  # or LoadBalancer
+      type: LoadBalancer  # or NodePort (then also use DaemonSet)
+      # set externalTrafficPolicy=Local to keep the source IP available in
+      # upstream services. Comes with tradeoff considerations, see
+      # documentation on "ingress" on docs.wire.com
       externalTrafficPolicy: Local
       nodePorts:
-        # The nginx instance is exposed on ports 31773 (https) and 31772 (http)
-        # on the node on which it runs. You should add a port-forwarding rule
-        # on the node or on the loadbalancer that forwards ports 443 and 80 to
-        # these respective ports.
+        # If you set service.type = NodePort, then the nginx controller instance
+        # is exposed on ports 31773 (https) and 31772 (http) on the node on
+        # which it runs. You should add a port-forwarding rule on the node or on
+        # the loadbalancer that forwards ports 443 and 80 to these respective
+        # ports.
         https: 31773
         http: 31772
     config:
-      # NOTE: These are some sane defaults (compliant to TR-02102-2), you may want to overrride them on your own installation
-      # For TR-02102-2 see https://www.bsi.bund.de/SharedDocs/Downloads/EN/BSI/Publications/TechGuidelines/TG02102/BSI-TR-02102-2.html
-      # As a Wire employee, for Wire-internal discussions and context see
-      # * https://wearezeta.atlassian.net/browse/FS-33
-      # * https://wearezeta.atlassian.net/browse/FS-444
+      # NOTE: These are some sane defaults (compliant to TR-02102-2), you may
+      # want to overrride them on your own installation For TR-02102-2 see
+      # https://www.bsi.bund.de/SharedDocs/Downloads/EN/BSI/Publications/TechGuidelines/TG02102/BSI-TR-02102-2.html
+      # As a Wire employee, for Wire-internal discussions and context see *
+      # https://wearezeta.atlassian.net/browse/FS-33 *
+      # https://wearezeta.atlassian.net/browse/FS-444
       ssl-protocols: "TLSv1.2 TLSv1.3"
       # override cipher suites used in TLS 1.2 (only, if TLS 1.2 is used)
       ssl-ciphers: "ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384"
diff --git a/docs/src/how-to/install/ingress.md b/docs/src/how-to/install/ingress.md
new file mode 100644
index 0000000000..14a0f3dd8b
--- /dev/null
+++ b/docs/src/how-to/install/ingress.md
@@ -0,0 +1,97 @@
+# Ingress traffic to wire-server (ingress-nginx-controller)
+
+*at the time of writing (2023-03), this section assumes you use a kubernetes
+version 1.23 or above (tested with 1.26)*
+
+## Installing in a cloud-like environment
+
+Install the ingress controller chart in your helmfile with the defaults, simply
+like this:
+
+```yaml
+# helmfile.yaml
+repositories:
+  - name: wire
+    url: 'https://s3-eu-west-1.amazonaws.com/public.wire.com/charts'
+
+releases:
+  - name: 'ingress-nginx-controller'
+    namespace: 'wire'
+    chart: 'wire/ingress-nginx-controller'
+    version: 'CHANGE_ME'
+```
+
+By default, the `wire/ingress-nginx-controller` chart will create a `Deployment`
+with services of type `LoadBalancer`, where your kubernetes installation needs
+to support dynamic LoadBalancers. If this is not possible, read the next section.
+
+By default three pods will come up and external traffic will be load balanced into these
+three pods, which will also do TLS termination and forward traffic to upstream
+services (`nginz` and others).
+
+To inspect default TLS settings, see TODO
+
+## Installing on bare-metal without dynamic load balancer support
+
+In case you cannot create a `kind: service` of `type: LoadBalancer`, then you
+can fall back to manually ensure traffic reaches your installation:
+
+```yaml
+# helmfile.yaml
+releases:
+  - name: 'ingress-nginx-controller'
+    namespace: 'wire'
+    chart: 'wire/ingress-nginx-controller'
+    version: 'CHANGE_ME'
+    values:
+      - './helm_vars/ingress-nginx-controller/values.yaml'
+```
+
+Create this file with the following override values:
+
+```yaml
+# helm_vars/ingress-nginx-controller/values.yaml
+ingress-nginx:
+  controller:
+    kind: DaemonSet
+    service:
+      type: NodePort
+```
+
+Then, on each of your kubernetes worker nodes, two ports are exposed: ports
+31773 (https) and 31772 (http)
+
+You should add a port-forwarding rule on the node or on the loadbalancer that
+forwards ports 443 and 80 to these respective ports. Any traffic hitting the http port is simply getting a http 30x redirect to https.
+
+Downsides of this approach: The NodePort approach always requires manual configuration of some external load balancer/firewall to round-robin between node IPs and is error-prone. It's also a bit annoying to have to decide on some global ports that may not be used otherwise.
+
+Most managed K8s clusters have support for LoadBalancers, you can also get this for your own clusters in hcloud etc. It's even possible to do it for pure bare metal, without any "load balancer hardware", by using BGP or some leadership election over who's announcing the "load balancer ip" via ARP (https://metallb.universe.tf/configuration/_advanced_l2_configuration/).
+
+### Using NodePort (not the default) with externalTrafficPolicy=Local (the default)
+
+Normally, NodePort will listen to traffic on all nodes, and uses kube-proxy
+to redirect to the node that actually runs ingress-nginx-controller. However
+one problem with this is that this traffic is NAT'ed. This means that nginx
+will not have access to the source IP address from which the request
+originated.  We want to have this source IP address for potentially logging
+and rate-limiting based on it. By setting externalTrafficPolicy: local,
+nodes will no longer forward requests to other nodes if they receive a
+request that they themselves can not handle. Upside is that the traffic is
+now not NAT'ed anymore, and we get access to the source IP address. Downside
+is that you need to know beforehand which nodes run a certain pod. However,
+with kubernetes a pod can be rescheduled to any node at any time so we can
+not trust this.  We could do something with node affinities to decide apriori
+on what set of nodes will be publicly reachable and make sure the nginx
+controller pods are only ran on there but for now that sounds a bit overkill.
+Instead, we just simply run the ingress controller on each node using a
+daemonset. This means that any node in the cluster can receive requests and
+redirect them to the correct service, whilst maintaining the source ip
+address. The ingress controller is sort of taking over the role of what
+kube-proxy was doing before.
+More information:
+- https://kubernetes.io/docs/tutorials/services/source-ip/#source-ip-for-services-with-typenodeport
+- https://kubernetes.github.io/ingress-nginx/deploy/baremetal/
+
+There are also downsides to setting `externalTrafficPolicy: Local`, please look at the [following blog post](https://www.asykim.com/blog/deep-dive-into-kubernetes-external-traffic-policies), which very clearly explains the upsides and
+downsides of this setting

From 9d74921e695b3ab260d548d5ad8e5e23ecf38815 Mon Sep 17 00:00:00 2001
From: jschaul <jschaul@users.noreply.github.com>
Date: Wed, 22 Mar 2023 15:06:18 +0100
Subject: [PATCH 38/40] tweak documentation

---
 docs/src/how-to/install/index.md   | 1 +
 docs/src/how-to/install/ingress.md | 7 ++++++-
 docs/src/how-to/install/tls.md     | 2 +-
 3 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/docs/src/how-to/install/index.md b/docs/src/how-to/install/index.md
index b45b694832..f312f8e332 100644
--- a/docs/src/how-to/install/index.md
+++ b/docs/src/how-to/install/index.md
@@ -19,6 +19,7 @@ Infrastructure configuration <infrastructure-configuration>
 How to monitor wire-server <monitoring>
 How to see centralized logs for wire-server <logging>
 
+Ingress-controller (getting traffic in) <ingress>
 Web app settings <web-app-settings>
 sft
 restund
diff --git a/docs/src/how-to/install/ingress.md b/docs/src/how-to/install/ingress.md
index 14a0f3dd8b..03a4329052 100644
--- a/docs/src/how-to/install/ingress.md
+++ b/docs/src/how-to/install/ingress.md
@@ -19,6 +19,11 @@ releases:
     namespace: 'wire'
     chart: 'wire/ingress-nginx-controller'
     version: 'CHANGE_ME'
+
+# charts wire-server and nginx-ingress-services also need to be installed, see other
+# documentation
+#  - name: ...
+#    chart: ...
 ```
 
 By default, the `wire/ingress-nginx-controller` chart will create a `Deployment`
@@ -29,7 +34,7 @@ By default three pods will come up and external traffic will be load balanced in
 three pods, which will also do TLS termination and forward traffic to upstream
 services (`nginz` and others).
 
-To inspect default TLS settings, see TODO
+To inspect default TLS settings, see [defaults in the latest code](https://github.com/wireapp/wire-server/blob/develop/charts/ingress-nginx-controller/values.yaml) and also see {ref}`tls`.
 
 ## Installing on bare-metal without dynamic load balancer support
 
diff --git a/docs/src/how-to/install/tls.md b/docs/src/how-to/install/tls.md
index f3a044597a..2de626d705 100644
--- a/docs/src/how-to/install/tls.md
+++ b/docs/src/how-to/install/tls.md
@@ -25,7 +25,7 @@ Therefore it is not necessary to add them to openssl based configurations.
 
 ## Ingress Traffic (wire-server)
 
-The list of TLS ciphers for incoming requests is limited by default to the [following](https://github.com/wireapp/wire-server/blob/master/charts/nginx-ingress-controller/values.yaml#L7) (for general server-certificates, both for federation and client API), and can be overridden on your installation if needed.
+The list of TLS ciphers for incoming requests is limited by default to the [following](https://github.com/wireapp/wire-server/blob/master/charts/ingress-nginx-controller/values.yaml#L41-45) (for general server-certificates, both for federation and client API), and can be overridden on your installation if needed.
 
 ## Egress Traffic (wire-server/federation)
 

From 5eba47abbdd71c57db29e7ce82b905f1af3310fe Mon Sep 17 00:00:00 2001
From: jschaul <jschaul@users.noreply.github.com>
Date: Wed, 22 Mar 2023 15:09:31 +0100
Subject: [PATCH 39/40] more docs tweaks

---
 charts/ingress-nginx-controller/values.yaml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/charts/ingress-nginx-controller/values.yaml b/charts/ingress-nginx-controller/values.yaml
index 80eb20ed9e..d8fd654f93 100644
--- a/charts/ingress-nginx-controller/values.yaml
+++ b/charts/ingress-nginx-controller/values.yaml
@@ -6,7 +6,8 @@
 # swapped words 'nginx' and 'ingress'), where we assume no load balancer support
 # and instead expose NodePorts on ports 31773 and 31772, and where you need to
 # ensure traffic gets to these ports in another way; then please read the
-# documentation on https://docs.wire.com (or search for "ingress")
+# documentation on https://docs.wire.com/how-to/install/ingress.html (or go to
+# https://docs.wire.com and search for "ingress-nginx-controller")
 #
 # See
 # https://github.com/kubernetes/ingress-nginx/blob/main/charts/ingress-nginx/values.yaml

From 344d6e299abc6436ad51aca63445eb5d5d8904a3 Mon Sep 17 00:00:00 2001
From: jschaul <jschaul@users.noreply.github.com>
Date: Wed, 22 Mar 2023 15:12:03 +0100
Subject: [PATCH 40/40] link to docs in changelog

---
 changelog.d/0-release-notes/ingress-controller | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/changelog.d/0-release-notes/ingress-controller b/changelog.d/0-release-notes/ingress-controller
index 85c52487f6..1f1f7ca5be 100644
--- a/changelog.d/0-release-notes/ingress-controller
+++ b/changelog.d/0-release-notes/ingress-controller
@@ -33,3 +33,5 @@ releases:
 +    chart: 'wire/ingress-nginx-controller'
      version: 'CHANGE_ME'
 ```
+
+For more information read the documentation under https://docs.wire.com/how-to/install/ingress.html (or go to https://docs.wire.com and search for "ingress-nginx-controller")