diff --git a/Makefile b/Makefile index 20d1110d9c..f32a11814e 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ DOCKER_TAG ?= $(USER) # default helm chart version must be 0.0.42 for local development (because 42 is the answer to the universe and everything) HELM_SEMVER ?= 0.0.42 # The list of helm charts needed on internal kubernetes testing environments -CHARTS_INTEGRATION := wire-server databases-ephemeral redis-cluster fake-aws nginx-ingress-controller nginx-ingress-services fluent-bit kibana sftd restund coturn +CHARTS_INTEGRATION := wire-server databases-ephemeral redis-cluster fake-aws ingress-nginx-controller nginx-ingress-controller nginx-ingress-services fluent-bit kibana sftd restund coturn # The list of helm charts to publish on S3 # FUTUREWORK: after we "inline local subcharts", # (e.g. move charts/brig to charts/wire-server/brig) @@ -17,7 +17,7 @@ CHARTS_RELEASE := wire-server redis-ephemeral redis-cluster databases-ephemeral fake-aws fake-aws-s3 fake-aws-sqs aws-ingress fluent-bit kibana backoffice \ calling-test demo-smtp elasticsearch-curator elasticsearch-external \ elasticsearch-ephemeral minio-external cassandra-external \ -nginx-ingress-controller nginx-ingress-services reaper sftd restund coturn \ +nginx-ingress-controller ingress-nginx-controller nginx-ingress-services reaper sftd restund coturn \ inbucket k8ssandra-test-cluster postgresql KIND_CLUSTER_NAME := wire-server HELM_PARALLELISM ?= 1 # 1 for sequential tests; 6 for all-parallel tests diff --git a/changelog.d/0-release-notes/ingress-controller b/changelog.d/0-release-notes/ingress-controller new file mode 100644 index 0000000000..1f1f7ca5be --- /dev/null +++ b/changelog.d/0-release-notes/ingress-controller @@ -0,0 +1,37 @@ +New 'ingress-nginx-controller' wrapper chart compatible with kubernetes versions [1.23 - 1.26]. The old one 'nginx-ingress-controller' (compatible only up to k8s 1.19) is now DEPRECATED. +We advise to upgrade your version of kubernetes in use to 1.23 or higher (we tested on kubernetes version 1.26), and to make use of the new ingress controller chart. Main features: +- up-to-date nginx version ('1.21.6') +- TLS 1.3 support (including allowing specifying which cipher suites to use) +- security fixes +- no more accidental logging of Wire access tokens under specific circumstances + +The 'kind: Ingress' resources installed via 'nginx-ingress-services' chart remain compatible with both the old and the new ingress controller, and k8s versions [1.18 - 1.26]. In case you upgrade an existing kubernetes cluster (not recommended), you may need to first uninstall the old controller before installing the new controller chart. + +In case you have custom overrides, you need to modify the directory name and top-level configuration key: + +```diff +# If you have overrides for the controller chart (such as cipher suites), ensure to rename file and top-level key: +-# nginx-ingress-controller/values.yaml ++# ingress-nginx-controller/values.yaml +-nginx-ingress: ++ingress-nginx: + controller: + # ... +``` + +and double-check if all overrides you use are indeed provided under the same name by the upstream chart. See also the default overrides in [the default values.yaml](https://github.com/wireapp/wire-server/blob/develop/charts/ingress-nginx-controller/values.yaml). + +In case you use helmfile change your ingress controller like this: + +```diff +# helmfile.yaml +releases: +- - name: 'nginx-ingress-controller' ++ - name: 'ingress-nginx-controller' + namespace: 'wire' +- chart: 'wire/nginx-ingress-controller' ++ chart: 'wire/ingress-nginx-controller' + version: 'CHANGE_ME' +``` + +For more information read the documentation under https://docs.wire.com/how-to/install/ingress.html (or go to https://docs.wire.com and search for "ingress-nginx-controller") diff --git a/changelog.d/5-internal/ingress-controller b/changelog.d/5-internal/ingress-controller new file mode 100644 index 0000000000..7d79310d91 --- /dev/null +++ b/changelog.d/5-internal/ingress-controller @@ -0,0 +1,5 @@ +- integration tests on CI will use either the old or the new ingress controller; depending on which kubernetes version they run on. +- upgrade `kubectl` to default from the nixpkgs channel (currently `1.26`) by removing the manual version pin on 1.19 +- upgrade `helmfile` to default from the nixpkgs channel by removing the manual version pin +- upgrade `helm` to default from the nixpkgs channel by removing the manual version pin +- add `kubelogin-oidc` so the kubectl in this environment can also talk to kubernetes clusters using OIDC diff --git a/charts/ingress-nginx-controller/Chart.yaml b/charts/ingress-nginx-controller/Chart.yaml new file mode 100644 index 0000000000..479a9b5d90 --- /dev/null +++ b/charts/ingress-nginx-controller/Chart.yaml @@ -0,0 +1,8 @@ +apiVersion: v2 +description: A Helm chart for an ingress controller (using nginx) on Kubernetes +name: ingress-nginx-controller +version: 0.0.42 +dependencies: +- name: ingress-nginx + version: 4.5.2 # k8s compatibility [1.23 - 1.26] + repository: https://kubernetes.github.io/ingress-nginx diff --git a/charts/ingress-nginx-controller/values.yaml b/charts/ingress-nginx-controller/values.yaml new file mode 100644 index 0000000000..d8fd654f93 --- /dev/null +++ b/charts/ingress-nginx-controller/values.yaml @@ -0,0 +1,58 @@ +# The following defaults apply to a cloud-like setup (in which you can ask your +# kubernetes installation to give you a LoadBalancer setup). +# +# If you are on bare metal and wish an installation similiar in spirit as the +# older similarly named wrapper chart 'nginx-ingress-controller' (note the +# swapped words 'nginx' and 'ingress'), where we assume no load balancer support +# and instead expose NodePorts on ports 31773 and 31772, and where you need to +# ensure traffic gets to these ports in another way; then please read the +# documentation on https://docs.wire.com/how-to/install/ingress.html (or go to +# https://docs.wire.com and search for "ingress-nginx-controller") +# +# See +# https://github.com/kubernetes/ingress-nginx/blob/main/charts/ingress-nginx/values.yaml +# for all possible values to override. +ingress-nginx: + controller: + enableTopologyAwareRouting: true + # Use kind: `DaemonSet` (when using NodePort) or `Deployment` (when using + # LoadBalancer) + kind: Deployment + service: + type: LoadBalancer # or NodePort (then also use DaemonSet) + # set externalTrafficPolicy=Local to keep the source IP available in + # upstream services. Comes with tradeoff considerations, see + # documentation on "ingress" on docs.wire.com + externalTrafficPolicy: Local + nodePorts: + # If you set service.type = NodePort, then the nginx controller instance + # is exposed on ports 31773 (https) and 31772 (http) on the node on + # which it runs. You should add a port-forwarding rule on the node or on + # the loadbalancer that forwards ports 443 and 80 to these respective + # ports. + https: 31773 + http: 31772 + config: + # NOTE: These are some sane defaults (compliant to TR-02102-2), you may + # want to overrride them on your own installation For TR-02102-2 see + # https://www.bsi.bund.de/SharedDocs/Downloads/EN/BSI/Publications/TechGuidelines/TG02102/BSI-TR-02102-2.html + # As a Wire employee, for Wire-internal discussions and context see * + # https://wearezeta.atlassian.net/browse/FS-33 * + # https://wearezeta.atlassian.net/browse/FS-444 + ssl-protocols: "TLSv1.2 TLSv1.3" + # override cipher suites used in TLS 1.2 (only, if TLS 1.2 is used) + ssl-ciphers: "ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384" + # override cipher suites used in TLS 1.3 (only, if TLS 1.3 is used) + server-snippet: "ssl_conf_command Ciphersuites TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384;" + # used to be called http2-max-(header|field)-size, removed in controller v1.3 + large-client-header-buffers: "16 32k" + proxy-buffer-size: "16k" + proxy-body-size: "1024m" + hsts-max-age: "31536000" + # Override log format to remove logging access tokens: + # removes 'request_query: "$args"', since it can include '?access_token=...' + # (sometimes sent for assets and websocket establishments) + # We do not wish to log these (SEC-47) + # Also add ssl/tls protocol/cipher to gain some observability here (can we turn off TLS 1.2?) + log-format-escape-json: true + log-format-upstream: '{"bytes_sent": "$bytes_sent", "duration": "$request_time", "http_referrer": "$http_referer", "http_user_agent": "$http_user_agent", "method": "$request_method", "path": "$uri", "remote_addr": "$proxy_protocol_addr", "remote_user": "$remote_user", "request_id": "$req_id", "request_length": "$request_length", "request_proto": "$server_protocol", "request_time": "$request_time", "status": "$status", "time": "$time_iso8601", "tls_cipher": "$ssl_cipher", "tls_protocol": "$ssl_protocol", "vhost": "$host", "x_forwarded_for": "$proxy_add_x_forwarded_for"}' diff --git a/charts/nginx-ingress-controller/Chart.yaml b/charts/nginx-ingress-controller/Chart.yaml index 96f94a8958..3177840c9c 100644 --- a/charts/nginx-ingress-controller/Chart.yaml +++ b/charts/nginx-ingress-controller/Chart.yaml @@ -1,4 +1,5 @@ apiVersion: v1 -description: A Helm chart for an ingress controller (using nginx) on Kubernetes +description: ingress-controller. DEPRECATED. Use ingress-nginx-controller chart instead. name: nginx-ingress-controller version: 0.0.42 +deprecated: true diff --git a/charts/nginx-ingress-controller/README.md b/charts/nginx-ingress-controller/README.md index 11a8508044..b0a4d744b1 100644 --- a/charts/nginx-ingress-controller/README.md +++ b/charts/nginx-ingress-controller/README.md @@ -1,3 +1,5 @@ +WARNING: deprecated. Use ingress-nginx-controller instead, if possible. + This deploys a single ingress controller - ideally, you want this on a separate, shared namespace since controllers listen on all namespaces by default (you can also modify that but it's generally discouraged). It is mostly a wrapper of the [nginx-ingress](https://github.com/helm/charts/blob/master/stable/nginx-ingress/README.md) with some other defaults that make sense for our use case(s). diff --git a/charts/nginx-ingress-controller/values.yaml b/charts/nginx-ingress-controller/values.yaml index 6063fa239a..7d1bc21c93 100644 --- a/charts/nginx-ingress-controller/values.yaml +++ b/charts/nginx-ingress-controller/values.yaml @@ -43,6 +43,7 @@ nginx-ingress: # downsides of this setting # https://www.asykim.com/blog/deep-dive-into-kubernetes-external-traffic-policies kind: DaemonSet + ingressClass: nginx # By default, each node will now be configured to accept ingress traffic. You should add # all the nodes to your external load balancer, or add them to DNS records. # diff --git a/charts/nginx-ingress-services/README.md b/charts/nginx-ingress-services/README.md index 50eb736fa8..2dadd73b38 100644 --- a/charts/nginx-ingress-services/README.md +++ b/charts/nginx-ingress-services/README.md @@ -1,5 +1,5 @@ This helm chart is a helper to set up needed services, ingresses and (likely) secrets to access your cluster. -It will _NOT_ deploy an ingress controller! Ensure you already have one on your cluster - or have a look at our [nginx-ingress-controller](../nginx-ingress-controller/README.md) +It will _NOT_ deploy an ingress controller! Ensure you already have one on your cluster - or have a look at our [ingress-nginx-controller](../ingress-nginx-controller/README.md) If tls.enabled == true, then you need to supply 2 variables, `tlsWildcardCert` and `tlsWildcardKey` that could either be supplied as plain text in the form of a `-f path/to/secrets.yaml`, like this: @@ -28,16 +28,14 @@ Q: My ingress keeps serving "Kubernetes Ingress Controller Fake Certificate"!! A: Ensure that your certificate is _valid_ and has _not expired_; trying to serve expired certificates will silently fail and the nginx ingress will simply fallback to the default certificate. - ## About cert-manager ### Prerequisites -* `cert-manager` and its CRDs have to be installed upfront, +* `cert-manager` and its CRDs have to be installed upfront, e.g. `helm upgrade --install -n cert-manager-ns --set 'installCRDs=true' cert-manager jetstack/cert-manager`, because upstream decided that this is the way (https://github.com/jetstack/cert-manager/pull/2964) - ### What does this chart do? * define `Ingress` for various services and their corresponding FQDNS @@ -45,10 +43,4 @@ A: Ensure that your certificate is _valid_ and has _not expired_; trying to serv *cert-manager* take care of this * [optional] configure an *Issuer* to issue ACME HTTP01 certificates provided by Letsencrypt * [optional] define a *Certificate* representation that causes *cert-manager* to issue a - certificate that is then used by `Ingress` - - -### Todo when introducing support for K8s >= 1.15 - -* the `apiVersion` of all resources based on cert-manager's CRDs, namely `./templates/issuer.yaml` and - `./templates/certificate.yaml`, has to be changed to `cert-manager.io/v1alpha3` + certificate that is then used by `Ingress` diff --git a/charts/nginx-ingress-services/templates/_helpers.tpl b/charts/nginx-ingress-services/templates/_helpers.tpl index 32f4467617..7b0faa4711 100644 --- a/charts/nginx-ingress-services/templates/_helpers.tpl +++ b/charts/nginx-ingress-services/templates/_helpers.tpl @@ -86,3 +86,7 @@ Returns the Letsencrypt API server URL based on whether testMode is enabled or d {{- define "ingress.supportsPathType" -}} {{- or (eq (include "ingress.isStable" .) "true") (and (eq (include "ingress.apiVersion" .) "networking.k8s.io/v1beta1") (semverCompare ">= 1.18-0" (include "kubeVersion" .))) -}} {{- end -}} + +{{- define "integrationTestHelperNewLabels" -}} + {{- (semverCompare ">= 1.23-0" (include "kubeVersion" .)) -}} +{{- end -}} diff --git a/charts/nginx-ingress-services/templates/federation-test-helper.yaml b/charts/nginx-ingress-services/templates/federation-test-helper.yaml index 4c06577052..0fb621e34d 100644 --- a/charts/nginx-ingress-services/templates/federation-test-helper.yaml +++ b/charts/nginx-ingress-services/templates/federation-test-helper.yaml @@ -1,3 +1,4 @@ +{{- $newLabels := eq (include "integrationTestHelperNewLabels" .) "true" -}} # Assumes that the controller is deployed in the same namespace. Only used for # enabling discovery by creating SRV records while running integration tests. {{- if (and .Values.federator.enabled .Values.federator.integrationTestHelper) }} @@ -13,7 +14,12 @@ spec: protocol: TCP targetPort: https selector: + {{- if $newLabels }} + app.kubernetes.io/component: controller + app.kubernetes.io/name: ingress-nginx + {{- else }} app: nginx-ingress component: controller + {{- end }} type: ClusterIP {{- end }} diff --git a/charts/nginx-ingress-services/templates/ingress.yaml b/charts/nginx-ingress-services/templates/ingress.yaml index 0314ee9513..6b19cdd090 100644 --- a/charts/nginx-ingress-services/templates/ingress.yaml +++ b/charts/nginx-ingress-services/templates/ingress.yaml @@ -5,7 +5,7 @@ kind: Ingress metadata: name: nginx-ingress annotations: - kubernetes.io/ingress.class: "nginx" + kubernetes.io/ingress.class: "{{ .Values.config.ingressClass }}" spec: # This assumes you have created the given cert (see secret.yaml) # https://github.com/kubernetes/ingress-nginx/blob/master/docs/examples/PREREQUISITES.md#tls-certificates diff --git a/charts/nginx-ingress-services/templates/ingress_federator.yaml b/charts/nginx-ingress-services/templates/ingress_federator.yaml index 8d52ff9a33..0318d9ce5b 100644 --- a/charts/nginx-ingress-services/templates/ingress_federator.yaml +++ b/charts/nginx-ingress-services/templates/ingress_federator.yaml @@ -8,7 +8,7 @@ kind: Ingress metadata: name: federator-ingress annotations: - kubernetes.io/ingress.class: "nginx" + kubernetes.io/ingress.class: "{{ .Values.config.ingressClass }}" nginx.ingress.kubernetes.io/ssl-redirect: "true" nginx.ingress.kubernetes.io/backend-protocol: "HTTP" nginx.ingress.kubernetes.io/auth-tls-verify-client: "on" diff --git a/charts/nginx-ingress-services/values.yaml b/charts/nginx-ingress-services/values.yaml index b76a61390d..259d80540f 100644 --- a/charts/nginx-ingress-services/values.yaml +++ b/charts/nginx-ingress-services/values.yaml @@ -91,8 +91,9 @@ service: accountPages: externalPort: 8080 +config: + ingressClass: "nginx" # You will need to supply some DNS names, namely -# config: # dns: # https: nginz-https. # ssl: nginz-ssl. # For websockets diff --git a/docs/src/how-to/install/index.md b/docs/src/how-to/install/index.md index b45b694832..f312f8e332 100644 --- a/docs/src/how-to/install/index.md +++ b/docs/src/how-to/install/index.md @@ -19,6 +19,7 @@ Infrastructure configuration How to monitor wire-server How to see centralized logs for wire-server +Ingress-controller (getting traffic in) Web app settings sft restund diff --git a/docs/src/how-to/install/ingress.md b/docs/src/how-to/install/ingress.md new file mode 100644 index 0000000000..03a4329052 --- /dev/null +++ b/docs/src/how-to/install/ingress.md @@ -0,0 +1,102 @@ +# Ingress traffic to wire-server (ingress-nginx-controller) + +*at the time of writing (2023-03), this section assumes you use a kubernetes +version 1.23 or above (tested with 1.26)* + +## Installing in a cloud-like environment + +Install the ingress controller chart in your helmfile with the defaults, simply +like this: + +```yaml +# helmfile.yaml +repositories: + - name: wire + url: 'https://s3-eu-west-1.amazonaws.com/public.wire.com/charts' + +releases: + - name: 'ingress-nginx-controller' + namespace: 'wire' + chart: 'wire/ingress-nginx-controller' + version: 'CHANGE_ME' + +# charts wire-server and nginx-ingress-services also need to be installed, see other +# documentation +# - name: ... +# chart: ... +``` + +By default, the `wire/ingress-nginx-controller` chart will create a `Deployment` +with services of type `LoadBalancer`, where your kubernetes installation needs +to support dynamic LoadBalancers. If this is not possible, read the next section. + +By default three pods will come up and external traffic will be load balanced into these +three pods, which will also do TLS termination and forward traffic to upstream +services (`nginz` and others). + +To inspect default TLS settings, see [defaults in the latest code](https://github.com/wireapp/wire-server/blob/develop/charts/ingress-nginx-controller/values.yaml) and also see {ref}`tls`. + +## Installing on bare-metal without dynamic load balancer support + +In case you cannot create a `kind: service` of `type: LoadBalancer`, then you +can fall back to manually ensure traffic reaches your installation: + +```yaml +# helmfile.yaml +releases: + - name: 'ingress-nginx-controller' + namespace: 'wire' + chart: 'wire/ingress-nginx-controller' + version: 'CHANGE_ME' + values: + - './helm_vars/ingress-nginx-controller/values.yaml' +``` + +Create this file with the following override values: + +```yaml +# helm_vars/ingress-nginx-controller/values.yaml +ingress-nginx: + controller: + kind: DaemonSet + service: + type: NodePort +``` + +Then, on each of your kubernetes worker nodes, two ports are exposed: ports +31773 (https) and 31772 (http) + +You should add a port-forwarding rule on the node or on the loadbalancer that +forwards ports 443 and 80 to these respective ports. Any traffic hitting the http port is simply getting a http 30x redirect to https. + +Downsides of this approach: The NodePort approach always requires manual configuration of some external load balancer/firewall to round-robin between node IPs and is error-prone. It's also a bit annoying to have to decide on some global ports that may not be used otherwise. + +Most managed K8s clusters have support for LoadBalancers, you can also get this for your own clusters in hcloud etc. It's even possible to do it for pure bare metal, without any "load balancer hardware", by using BGP or some leadership election over who's announcing the "load balancer ip" via ARP (https://metallb.universe.tf/configuration/_advanced_l2_configuration/). + +### Using NodePort (not the default) with externalTrafficPolicy=Local (the default) + +Normally, NodePort will listen to traffic on all nodes, and uses kube-proxy +to redirect to the node that actually runs ingress-nginx-controller. However +one problem with this is that this traffic is NAT'ed. This means that nginx +will not have access to the source IP address from which the request +originated. We want to have this source IP address for potentially logging +and rate-limiting based on it. By setting externalTrafficPolicy: local, +nodes will no longer forward requests to other nodes if they receive a +request that they themselves can not handle. Upside is that the traffic is +now not NAT'ed anymore, and we get access to the source IP address. Downside +is that you need to know beforehand which nodes run a certain pod. However, +with kubernetes a pod can be rescheduled to any node at any time so we can +not trust this. We could do something with node affinities to decide apriori +on what set of nodes will be publicly reachable and make sure the nginx +controller pods are only ran on there but for now that sounds a bit overkill. +Instead, we just simply run the ingress controller on each node using a +daemonset. This means that any node in the cluster can receive requests and +redirect them to the correct service, whilst maintaining the source ip +address. The ingress controller is sort of taking over the role of what +kube-proxy was doing before. +More information: +- https://kubernetes.io/docs/tutorials/services/source-ip/#source-ip-for-services-with-typenodeport +- https://kubernetes.github.io/ingress-nginx/deploy/baremetal/ + +There are also downsides to setting `externalTrafficPolicy: Local`, please look at the [following blog post](https://www.asykim.com/blog/deep-dive-into-kubernetes-external-traffic-policies), which very clearly explains the upsides and +downsides of this setting diff --git a/docs/src/how-to/install/tls.md b/docs/src/how-to/install/tls.md index f3a044597a..2de626d705 100644 --- a/docs/src/how-to/install/tls.md +++ b/docs/src/how-to/install/tls.md @@ -25,7 +25,7 @@ Therefore it is not necessary to add them to openssl based configurations. ## Ingress Traffic (wire-server) -The list of TLS ciphers for incoming requests is limited by default to the [following](https://github.com/wireapp/wire-server/blob/master/charts/nginx-ingress-controller/values.yaml#L7) (for general server-certificates, both for federation and client API), and can be overridden on your installation if needed. +The list of TLS ciphers for incoming requests is limited by default to the [following](https://github.com/wireapp/wire-server/blob/master/charts/ingress-nginx-controller/values.yaml#L41-45) (for general server-certificates, both for federation and client API), and can be overridden on your installation if needed. ## Egress Traffic (wire-server/federation) diff --git a/hack/bin/integration-setup-federation.sh b/hack/bin/integration-setup-federation.sh index f569928239..f624dc1167 100755 --- a/hack/bin/integration-setup-federation.sh +++ b/hack/bin/integration-setup-federation.sh @@ -20,13 +20,22 @@ ${DIR}/integration-cleanup.sh # script beforehand on all relevant charts to download the nested dependencies # (e.g. cassandra from underneath databases-ephemeral) echo "updating recursive dependencies ..." -charts=(fake-aws databases-ephemeral redis-cluster wire-server nginx-ingress-controller nginx-ingress-services) +charts=(fake-aws databases-ephemeral redis-cluster wire-server ingress-nginx-controller nginx-ingress-controller nginx-ingress-services) mkdir -p ~/.parallel && touch ~/.parallel/will-cite printf '%s\n' "${charts[@]}" | parallel -P "${HELM_PARALLELISM}" "$DIR/update.sh" "$CHARTS_DIR/{}" # FUTUREWORK: use helm functions instead, see https://wearezeta.atlassian.net/browse/SQPIT-723 echo "Generating self-signed certificates..." +KUBERNETES_VERSION_MAJOR="$(kubectl version -o json | jq -r .serverVersion.major)" +KUBERNETES_VERSION_MINOR="$(kubectl version -o json | jq -r .serverVersion.minor)" +export KUBERNETES_VERSION="$KUBERNETES_VERSION_MAJOR.$KUBERNETES_VERSION_MINOR" +if (( KUBERNETES_VERSION_MAJOR > 1 || KUBERNETES_VERSION_MAJOR == 1 && KUBERNETES_VERSION_MINOR >= 23 )); then + export INGRESS_CHART="ingress-nginx-controller" +else + export INGRESS_CHART="nginx-ingress-controller" +fi +echo "kubeVersion: $KUBERNETES_VERSION and ingress controller=$INGRESS_CHART" export NAMESPACE_1="$NAMESPACE" export FEDERATION_DOMAIN_BASE="$NAMESPACE_1.svc.cluster.local" export FEDERATION_DOMAIN_1="federation-test-helper.$FEDERATION_DOMAIN_BASE" diff --git a/hack/bin/integration-setup.sh b/hack/bin/integration-setup.sh index 59cf0e4f84..ed6be40c9e 100755 --- a/hack/bin/integration-setup.sh +++ b/hack/bin/integration-setup.sh @@ -14,10 +14,19 @@ HELM_PARALLELISM=${HELM_PARALLELISM:-1} "${DIR}/integration-cleanup.sh" echo "updating recursive dependencies ..." -charts=(fake-aws databases-ephemeral redis-cluster wire-server nginx-ingress-controller nginx-ingress-services) +charts=(fake-aws databases-ephemeral redis-cluster wire-server ingress-nginx-controller nginx-ingress-controller nginx-ingress-services) mkdir -p ~/.parallel && touch ~/.parallel/will-cite printf '%s\n' "${charts[@]}" | parallel -P "${HELM_PARALLELISM}" "$DIR/update.sh" "$CHARTS_DIR/{}" +KUBERNETES_VERSION_MAJOR="$(kubectl version -o json | jq -r .serverVersion.major)" +KUBERNETES_VERSION_MINOR="$(kubectl version -o json | jq -r .serverVersion.minor)" +export KUBERNETES_VERSION="$KUBERNETES_VERSION_MAJOR.$KUBERNETES_VERSION_MINOR" +if (( KUBERNETES_VERSION_MAJOR > 1 || KUBERNETES_VERSION_MAJOR == 1 && KUBERNETES_VERSION_MINOR >= 23 )); then + export INGRESS_CHART="ingress-nginx-controller" +else + export INGRESS_CHART="nginx-ingress-controller" +fi +echo "kubeVersion: $KUBERNETES_VERSION and ingress controller=$INGRESS_CHART" echo "Generating self-signed certificates..." export FEDERATION_DOMAIN_BASE="$NAMESPACE.svc.cluster.local" export FEDERATION_DOMAIN="federation-test-helper.$FEDERATION_DOMAIN_BASE" diff --git a/hack/bin/integration-teardown-federation.sh b/hack/bin/integration-teardown-federation.sh index 91897a6059..a439ab6219 100755 --- a/hack/bin/integration-teardown-federation.sh +++ b/hack/bin/integration-teardown-federation.sh @@ -12,6 +12,13 @@ export NAMESPACE_2="$NAMESPACE-fed2" export FEDERATION_DOMAIN_1="." export FEDERATION_DOMAIN_2="." +KUBERNETES_VERSION_MINOR="$(kubectl version -o json | jq -r .serverVersion.minor)" +if (( KUBERNETES_VERSION_MAJOR > 1 || KUBERNETES_VERSION_MAJOR == 1 && KUBERNETES_VERSION_MINOR >= 23 )); then + export INGRESS_CHART="ingress-nginx-controller" +else + export INGRESS_CHART="nginx-ingress-controller" +fi + . "$DIR/helm_overrides.sh" helmfile --file "${TOP_LEVEL}/hack/helmfile.yaml" destroy diff --git a/hack/bin/integration-teardown.sh b/hack/bin/integration-teardown.sh index cd82194c2b..60f1781b53 100755 --- a/hack/bin/integration-teardown.sh +++ b/hack/bin/integration-teardown.sh @@ -6,6 +6,12 @@ TOP_LEVEL="$DIR/../.." NAMESPACE=${NAMESPACE:-test-integration} # doesn't matter for destruction but needs to be set export FEDERATION_DOMAIN="." +KUBERNETES_VERSION_MINOR="$(kubectl version -o json | jq -r .serverVersion.minor)" +if (( KUBERNETES_VERSION_MAJOR > 1 || KUBERNETES_VERSION_MAJOR == 1 && KUBERNETES_VERSION_MINOR >= 23 )); then + export INGRESS_CHART="ingress-nginx-controller" +else + export INGRESS_CHART="nginx-ingress-controller" +fi set -ex diff --git a/hack/bin/set-helm-chart-version.sh b/hack/bin/set-helm-chart-version.sh index 759da9a218..00b838642e 100755 --- a/hack/bin/set-helm-chart-version.sh +++ b/hack/bin/set-helm-chart-version.sh @@ -12,7 +12,7 @@ tempfile=$(mktemp) function update_chart(){ chart_file=$1 - sed -e "s/version: .*/version: $target_version/g" "$chart_file" > "$tempfile" && mv "$tempfile" "$chart_file" + sed -e "s/^version: .*/version: $target_version/g" "$chart_file" > "$tempfile" && mv "$tempfile" "$chart_file" } function write_versions() { diff --git a/hack/bin/update.sh b/hack/bin/update.sh index 20d597a796..47ecefb2b0 100755 --- a/hack/bin/update.sh +++ b/hack/bin/update.sh @@ -29,6 +29,12 @@ helmDepUp () { helm dep up echo "... updating in $path done." fi + + if grep "dependencies:" Chart.yaml; then + echo "Updating dependencies (from Chart.yaml) in $path ..." + helm dep up + echo "... updating in $path done." + fi } helmDepUp "$dir" diff --git a/hack/helm_vars/ingress-nginx-controller/values.yaml.gotmpl b/hack/helm_vars/ingress-nginx-controller/values.yaml.gotmpl new file mode 100644 index 0000000000..d3c24971e6 --- /dev/null +++ b/hack/helm_vars/ingress-nginx-controller/values.yaml.gotmpl @@ -0,0 +1,19 @@ +ingress-nginx: + controller: + ingressClassResource: + name: "nginx-{{ .Release.Namespace }}" + # -- Is this ingressClass enabled or not + enabled: true + ingressClass: "nginx-{{ .Release.Namespace }}" + kind: Deployment + replicaCount: 1 + service: + nodePorts: + # choose a random free port + https: null + http: null + # in CI, do not use ValidatingWebhooks, as these, if not properly cleaned up + # (i.e. the ingress controller was deleted in another namespace but the webhook remains) + # prevent new kind:Ingress resources to be created in the cluster. + admissionWebhooks: + enabled: false diff --git a/hack/helm_vars/nginx-ingress-controller/values.yaml b/hack/helm_vars/nginx-ingress-controller/values.yaml.gotmpl similarity index 67% rename from hack/helm_vars/nginx-ingress-controller/values.yaml rename to hack/helm_vars/nginx-ingress-controller/values.yaml.gotmpl index 53b1c57506..10fd76e22b 100644 --- a/hack/helm_vars/nginx-ingress-controller/values.yaml +++ b/hack/helm_vars/nginx-ingress-controller/values.yaml.gotmpl @@ -1,6 +1,8 @@ nginx-ingress: controller: - kind: DaemonSet + kind: Deployment + replicaCount: 1 + ingressClass: "nginx-{{ .Release.Namespace }}" service: type: NodePort externalTrafficPolicy: Local diff --git a/hack/helm_vars/nginx-ingress-services/values.yaml b/hack/helm_vars/nginx-ingress-services/values.yaml.gotmpl similarity index 92% rename from hack/helm_vars/nginx-ingress-services/values.yaml rename to hack/helm_vars/nginx-ingress-services/values.yaml.gotmpl index 76aa0657e8..fd80c5ca43 100644 --- a/hack/helm_vars/nginx-ingress-services/values.yaml +++ b/hack/helm_vars/nginx-ingress-services/values.yaml.gotmpl @@ -9,6 +9,7 @@ tls: useCertManager: false config: + ingressClass: "nginx-{{ .Release.Namespace }}" dns: https: nginz-https.integration.example.com ssl: nginz-ssl.integration.example.com diff --git a/hack/helmfile-single.yaml b/hack/helmfile-single.yaml index 3a770ee146..bb9f4f5b2f 100644 --- a/hack/helmfile-single.yaml +++ b/hack/helmfile-single.yaml @@ -13,7 +13,9 @@ environments: values: - namespace: {{ requiredEnv "NAMESPACE" }} - federationDomain: {{ requiredEnv "FEDERATION_DOMAIN" }} + - ingressChart: {{ requiredEnv "INGRESS_CHART" }} - imagePullPolicy: Always + - redisStorageClass: hcloud-volumes repositories: - name: stable @@ -39,17 +41,17 @@ releases: values: - './helm_vars/redis-cluster/values.yaml.gotmpl' - - name: '{{ .Values.namespace }}-nginx-ingress-controller' + - name: '{{ .Values.namespace }}-ic' namespace: '{{ .Values.namespace }}' - chart: '../.local/charts/nginx-ingress-controller' + chart: '../.local/charts/{{ .Values.ingressChart }}' values: - - './helm_vars/nginx-ingress-controller/values.yaml' + - './helm_vars/{{ .Values.ingressChart }}/values.yaml.gotmpl' - - name: '{{ .Values.namespace }}-nginx-ingress-services' + - name: '{{ .Values.namespace }}-i' namespace: '{{ .Values.namespace }}' chart: '../.local/charts/nginx-ingress-services' values: - - './helm_vars/nginx-ingress-services/values.yaml' + - './helm_vars/nginx-ingress-services/values.yaml.gotmpl' - './helm_vars/nginx-ingress-services/certificates-namespace1.yaml' set: # Federation domain is also the SRV record created by the @@ -57,6 +59,8 @@ releases: # differ, so we don't make any silly assumptions in the code. - name: config.dns.federator value: {{ .Values.federationDomain }} + needs: + - '{{ .Values.namespace }}-ic' # Note that wire-server depends on databases-ephemeral being up; and in some # cases on nginx-ingress also being up. If installing helm charts in a diff --git a/hack/helmfile.yaml b/hack/helmfile.yaml index 6392d64a43..f9e608107e 100644 --- a/hack/helmfile.yaml +++ b/hack/helmfile.yaml @@ -18,6 +18,7 @@ environments: - federationDomain: {{ requiredEnv "FEDERATION_DOMAIN_1" }} - namespaceFed2: {{ requiredEnv "NAMESPACE_2" }} - federationDomainFed2: {{ requiredEnv "FEDERATION_DOMAIN_2" }} + - ingressChart: {{ requiredEnv "INGRESS_CHART" }} - imagePullPolicy: Always - redisStorageClass: hcloud-volumes kind: @@ -26,6 +27,7 @@ environments: - federationDomain: {{ requiredEnv "FEDERATION_DOMAIN_1" }} - namespaceFed2: {{ requiredEnv "NAMESPACE_2" }} - federationDomainFed2: {{ requiredEnv "FEDERATION_DOMAIN_2" }} + - ingressChart: {{ requiredEnv "INGRESS_CHART" }} - imagePullPolicy: Never - redisStorageClass: standard @@ -36,6 +38,9 @@ repositories: - name: bitnami url: 'https://charts.bitnami.com/bitnami' + - name: ingress + url: 'https://kubernetes.github.io/ingress-nginx' + releases: - name: '{{ .Values.namespace }}-fake-aws' namespace: '{{ .Values.namespace }}' @@ -69,23 +74,23 @@ releases: values: - './helm_vars/redis-cluster/values.yaml.gotmpl' - - name: '{{ .Values.namespace }}-nginx-ingress-controller' + - name: '{{ .Values.namespace }}-ic' namespace: '{{ .Values.namespace }}' - chart: '../.local/charts/nginx-ingress-controller' + chart: '../.local/charts/{{ .Values.ingressChart }}' values: - - './helm_vars/nginx-ingress-controller/values.yaml' + - './helm_vars/{{ .Values.ingressChart }}/values.yaml.gotmpl' - - name: '{{ .Values.namespace }}-nginx-ingress-controller-2' + - name: '{{ .Values.namespace }}-ic2' namespace: '{{ .Values.namespaceFed2 }}' - chart: '../.local/charts/nginx-ingress-controller' + chart: '../.local/charts/{{ .Values.ingressChart }}' values: - - './helm_vars/nginx-ingress-controller/values.yaml' + - './helm_vars/{{ .Values.ingressChart }}/values.yaml.gotmpl' - - name: '{{ .Values.namespace }}-nginx-ingress-services' + - name: '{{ .Values.namespace }}-i' namespace: '{{ .Values.namespace }}' chart: '../.local/charts/nginx-ingress-services' values: - - './helm_vars/nginx-ingress-services/values.yaml' + - './helm_vars/nginx-ingress-services/values.yaml.gotmpl' - './helm_vars/nginx-ingress-services/certificates-namespace1.yaml' set: # Federation domain is also the SRV record created by the @@ -93,12 +98,14 @@ releases: # differ, so we don't make any silly assumptions in the code. - name: config.dns.federator value: {{ .Values.federationDomain }} + needs: + - '{{ .Values.namespace }}-ic' - - name: '{{ .Values.namespace }}-nginx-ingress-services-2' + - name: '{{ .Values.namespace }}-i2' namespace: '{{ .Values.namespaceFed2 }}' chart: '../.local/charts/nginx-ingress-services' values: - - './helm_vars/nginx-ingress-services/values.yaml' + - './helm_vars/nginx-ingress-services/values.yaml.gotmpl' - './helm_vars/nginx-ingress-services/certificates-namespace2.yaml' set: # Federation domain is also the SRV record created by the @@ -106,9 +113,9 @@ releases: # differ, so we don't make any silly assumptions in the code. - name: config.dns.federator value: {{ .Values.federationDomainFed2 }} + needs: + - '{{ .Values.namespace }}-ic2' - #--------------------------------------------- - # # Note that wire-server depends on databases-ephemeral being up; and in some # cases on nginx-ingress also being up. If installing helm charts in a # parallel way, it's expected to see some wire-server pods (namely the diff --git a/nix/overlay.nix b/nix/overlay.nix index 767abcaa20..f5f8853331 100644 --- a/nix/overlay.nix +++ b/nix/overlay.nix @@ -88,36 +88,6 @@ self: super: { inherit (super) stdenv fetchurl; }; - helm = super.callPackage ./pkgs/helm { }; - - helmfile = staticBinary { - pname = "helmfile"; - version = "0.141.0"; - - darwinAmd64Url = "https://github.com/roboll/helmfile/releases/download/v0.141.0/helmfile_darwin_amd64"; - darwinAmd64Sha256 = "0szfd3vy6fzd5657079hz5vii86f9xkg3bdzp3g4knkcw5x1kpxy"; - - linuxAmd64Url = "https://github.com/roboll/helmfile/releases/download/v0.141.0/helmfile_linux_amd64"; - linuxAmd64Sha256 = "0f5d9w3qjvwip4qn79hsigwp8nbjpj58p289hww503j43wjyxx8r"; - - inherit (super) stdenv fetchurl; - }; - - kubectl = staticBinaryInTarball { - pname = "kubectl"; - version = "1.19.8"; - - darwinAmd64Url = "https://dl.k8s.io/v1.19.8/kubernetes-client-darwin-amd64.tar.gz"; - darwinAmd64Sha256 = "23b847bb8b545c748e9078e7660c654eef74d15ccab8696d294f3d6c619c788e"; - - linuxAmd64Url = "https://dl.k8s.io/v1.19.8/kubernetes-client-linux-amd64.tar.gz"; - linuxAmd64Sha256 = "8388ff8b5c676bdbb8fe07ef7077de937b0bf60154f302df5f248f38f95122aa"; - - binPath = "client/bin/kubectl"; - - inherit (super) stdenv fetchurl; - }; - kind = staticBinary { pname = "kind"; version = "0.11.0"; diff --git a/nix/pkgs/helm/default.nix b/nix/pkgs/helm/default.nix deleted file mode 100644 index 8b68403913..0000000000 --- a/nix/pkgs/helm/default.nix +++ /dev/null @@ -1,52 +0,0 @@ -# Copied from nixpkgs and modified because it seems too complicated to override -# buildGoModule packages. -{ lib, stdenv, buildGoModule, fetchFromGitHub, installShellFiles }: - -buildGoModule rec { - pname = "kubernetes-helm"; - version = "3.11.0-patched"; - - src = fetchFromGitHub { - owner = "wireapp"; - repo = "helm"; - rev = "949de3195be5b3d21ed707da18ee3bcb2a9a2af8"; - sha256 = "sha256-alyR6+gm7WEvFfJxHl9a0jpC3+457Kg6aRHcidA0RZg="; - }; - vendorSha256 = "sha256-LRMDrBSl5EGQqQt5FUU4JJHqdwfYt5qsVpe76jUQBVI="; - - subPackages = [ "cmd/helm" ]; - ldflags = [ - "-w" - "-s" - "-X helm.sh/helm/v3/internal/version.version=v${version}" - "-X helm.sh/helm/v3/internal/version.gitCommit=${src.rev}" - ]; - - preCheck = '' - # skipping version tests because they require dot git directory - substituteInPlace cmd/helm/version_test.go \ - --replace "TestVersion" "SkipVersion" - '' + lib.optionalString stdenv.isLinux '' - # skipping plugin tests on linux - substituteInPlace cmd/helm/plugin_test.go \ - --replace "TestPluginDynamicCompletion" "SkipPluginDynamicCompletion" \ - --replace "TestLoadPlugins" "SkipLoadPlugins" - substituteInPlace cmd/helm/helm_test.go \ - --replace "TestPluginExitCode" "SkipPluginExitCode" - ''; - - nativeBuildInputs = [ installShellFiles ]; - postInstall = '' - $out/bin/helm completion bash > helm.bash - $out/bin/helm completion zsh > helm.zsh - installShellCompletion helm.{bash,zsh} - ''; - - meta = with lib; { - homepage = "https://github.com/kubernetes/helm"; - description = "A package manager for kubernetes"; - mainProgram = "helm"; - license = licenses.asl20; - maintainers = with maintainers; [ rlupton20 edude03 saschagrunert Frostman Chili-Man techknowlogick ]; - }; -} diff --git a/nix/wire-server.nix b/nix/wire-server.nix index b1883e0405..c41815483a 100644 --- a/nix/wire-server.nix +++ b/nix/wire-server.nix @@ -300,12 +300,13 @@ let pkgs.gnused pkgs.parallel pkgs.ripgrep - pkgs.helm + pkgs.kubernetes-helm pkgs.helmfile pkgs.hlint (hlib.justStaticExecutables pkgs.haskellPackages.apply-refact) pkgs.jq pkgs.kubectl + pkgs.kubelogin-oidc pkgs.nixpkgs-fmt pkgs.ormolu pkgs.shellcheck