feat(base-cluster/ingress)!: add option traefik for ingress controller and make it default

charts/base-cluster/README.md.gotmpl

@@ -107,8 +107,16 @@ which is also supported by [cert-manager](https://cert-manager.io/docs/configura
 
 ### Component [ingress](#ingress)
 
-The included [`nginx` ingress-controller](https://docs.nginx.com/nginx-ingress-controller)
-only works for the `IngressClassName: nginx`.
+The chart supports two ingress controllers:
+
+1. [`nginx` ingress-controller](https://docs.nginx.com/nginx-ingress-controller) (default)
+   - Works with `IngressClassName: nginx` or if none is defined
+   - Provides built-in metrics and tracing support
+
+2. [`traefik`](https://traefik.io) (recommended)
+   - Works with `IngressClassName: ingress-controller` or if none is defined
+   - Provides built-in metrics and tracing support
+   - Also supports [Gateway API](https://gateway-api.sigs.k8s.io)
 
 #### TLS
 
@@ -122,7 +130,7 @@ only works for the `IngressClassName: nginx`.
 
 If you want to make sure that, in the event of a catastrophic failure, you keep the
 same IP address, you should roll this out, get the assigned IP
-(`kubectl -n ingress-nginx get svc ingress-nginx-controller -o jsonpath='{.status.loadBalancer.ingress}'`)
+(`kubectl -n ingress-nginx get svc ingress-nginx-controller -o jsonpath='{.status.loadBalancer.ingress}'` for nginx or `kubectl -n ingress get svc ingress-controller -o jsonpath='{.status.loadBalancer.ingress}'` for traefik)
 and set `.ingress.IP=<ip>` in the values. This makes sure the IP is kept in your
 project (may incur cost!), which means you can reuse it later or after recovery.
 
@@ -336,4 +344,53 @@ This also replaces `promtail` and the `otel-collector` with `alloy`, using
 <https://github.com/teutonet/teutonet-helm-charts/blob/main/charts/common/templates/_telemetry.tpl>
 makes this a drop-in change.
 
+### 8.x.x -> 9.0.0
+
+This release adds another option for ingress, [traefik](https://traefik.io)! 🎉
+
+If you have disabled ingress in your configuration, you need to update your
+values from:
+
+```yaml
+ingress:
+  enabled: false
+```
+
+to:
+
+```yaml
+ingress:
+  provider: none
+```
+
+If you are using ingress (the default), you need to either switch over to traefik
+or adjust your config to use nginx.
+But we do recommend using traefik, especially in light of <https://github.com/kubernetes/ingress-nginx/issues/13002>.
+
+To switch to traefik you don't need to do anything.
+
+This will delete the old service which in turn will get you a new IP.
+The `ingress-nginx` namespace will be deleted, so make sure you don't have any other
+stuff deployed there or adjust its [condition](https://github.com/teutonet/teutonet-helm-charts/tree/main/charts/base-cluster/#11412--property-base-cluster-configuration--global--namespaces--additionalproperties--condition)
+
+Using a [DNS Provider](#component-dns) will automatically update your DNS records.
+
+If you want to keep the same IP, do
+<https://github.com/teutonet/teutonet-helm-charts/tree/main/charts/base-cluster/#ip-address>
+beforehand.
+
+The switch will still create downtime, so be aware of that.
+
+In nginx it was possible to enable `allowNginxConfigurationSnippets` to add custom
+configuration to the nginx ingress controller.
+In traefik this is not possible, but you can use [gateway api](https://gateway-api.sigs.k8s.io)
+instead, making this agnostic.
+
+If you want to keep nginx, you need to configure the following;
+
+```yaml
+ingress:
+  provider: nginx
+```
+
 {{ .Files.Get "values.md"  }}

charts/base-cluster/ci/artifacthub-values-2.yaml

@@ -0,0 +1,6 @@
+global:
+  clusterName: test
+  baseDomain: example.com
+  serviceLevelAgreement: None
+ingress:
+  provider: traefik

charts/base-cluster/ci/artifacthub-values.yaml

@@ -53,7 +53,7 @@ kube-janitor:
 descheduler:
   enabled: true
 ingress:
-  enabled: true
+  provider: nginx
 reflector:
   enabled: true
 dns:

charts/base-cluster/ci/disabled-ingress-values.yaml

@@ -1,2 +1,2 @@
 ingress:
-  enabled: false
+  provider: none

charts/base-cluster/ci/traefik-ingress-values.yaml

@@ -0,0 +1,2 @@
+ingress:
+  provider: traefik

charts/base-cluster/templates/NOTES.txt

@@ -20,7 +20,7 @@
 ===
 You can access your grafana instance via
 
-{{- if and .Values.ingress.enabled .Values.monitoring.grafana.ingress.enabled .Values.certManager.email (or .Values.global.baseDomain .Values.monitoring.grafana.ingress.customDomain) }}
+{{- if and (ne .Values.ingress.provider "none") .Values.monitoring.grafana.ingress.enabled .Values.certManager.email (or .Values.global.baseDomain .Values.monitoring.grafana.ingress.customDomain) }}
 {{- printf "https://%s" (include "base-cluster.grafana.host" $) | nindent 2 }}
 {{- else }}
 {{- printf "$ kubectl -n monitoring port-forward svc/kube-prometheus-stack-grafana 3000:http-web" | nindent 2 }}

charts/base-cluster/templates/backup/velero.yaml

@@ -90,11 +90,11 @@ spec:
       uploaderType: restic
     metrics:
       serviceMonitor:
-        additionalLabels: {{- toYaml .Values.monitoring.labels | nindent 10 }}
+        additionalLabels: {{- include "common.tplvalues.render" (dict "value" .Values.monitoring.labels "context" .) | nindent 10 }}
         enabled: true
       prometheusRule:
         enabled: true
-        additionalLabels: {{- toYaml .Values.monitoring.labels | nindent 10 }}
+        additionalLabels: {{- include "common.tplvalues.render" (dict "value" .Values.monitoring.labels "context" .) | nindent 10 }}
         spec:
           - alert: VeleroBackupFailures
             annotations:

charts/base-cluster/templates/cert-manager/cert-manager.yaml

@@ -40,6 +40,12 @@ spec:
     extraArgs:
       - --dns01-recursive-nameservers={{- $nameservers | sortAlpha | join "," }}
     {{- end }}
+    {{- if eq .Values.ingress.provider "traefik" }}
+    config:
+      apiVersion: controller.config.cert-manager.io/v1alpha1
+      kind: ControllerConfiguration
+      enableGatewayAPI: true
+    {{- end }}
     resources: {{- include "common.resources" $.Values.certManager | nindent 6 }}
     replicaCount: 1
     securityContext: &securityContext
@@ -65,4 +71,4 @@ spec:
       enabled: {{ .Values.monitoring.prometheus.enabled }}
       servicemonitor:
         enabled: {{ .Values.monitoring.prometheus.enabled }}
-        labels: {{- toYaml .Values.monitoring.labels | nindent 10 }}
+        labels: {{- include "common.tplvalues.render" (dict "value" .Values.monitoring.labels "context" .) | nindent 10 }}

charts/base-cluster/templates/cert-manager/clusterissuer.yaml

@@ -1,4 +1,4 @@
-{{- if and .Values.certManager.email .Values.ingress.enabled }}
+{{- if and .Values.certManager.email (ne .Values.ingress.provider "none") }}
 {{- include "base-cluster.helm.resourceWithDependencies" (dict "name" "clusterissuer-letsencrypt-production" "resource" (include "base-cluster.cert-manager.clusterIssuer" (dict "name" "production" "url" "https://acme-v02.api.letsencrypt.org/directory" "context" $)) "dependencies" (dict "cert-manager" "cert-manager") "context" $ "additionalLabels" (dict "app.kubernetes.io/component" "cert-manager")) }}
 ---
 {{- include "base-cluster.helm.resourceWithDependencies" (dict "name" "clusterissuer-letsencrypt-staging" "resource" (include "base-cluster.cert-manager.clusterIssuer" (dict "name" "staging" "url" "https://acme-staging-v02.api.letsencrypt.org/directory" "context" $)) "dependencies" (dict "cert-manager" "cert-manager") "context" $ "additionalLabels" (dict "app.kubernetes.io/component" "cert-manager")) }}
@@ -33,7 +33,6 @@ spec:
       {{- end }}
       - http01:
           ingress:
-            class: nginx
             serviceType: ClusterIP
     privateKeySecretRef:
       name: letsencrypt-{{ .name }}-account

charts/base-cluster/templates/cert-manager/rules/certificate-expiration.yaml

@@ -9,7 +9,7 @@ metadata:
   name: certificate-expiration
   namespace: cert-manager
   labels: {{- include "common.labels.standard" . | nindent 4 }}
-    {{- with .Values.monitoring.labels }}{{ toYaml . | nindent 4 }}{{- end }}
+    {{- with .Values.monitoring.labels }}{{- include "common.tplvalues.render" (dict "value" . "context" $) | nindent 4 }}{{- end }}
     app.kubernetes.io/component: prometheus
     app.kubernetes.io/part-of: cert-manager
 spec:

charts/base-cluster/templates/descheduler/descheduler.yaml

@@ -44,7 +44,7 @@ spec:
       enabled: true
     serviceMonitor:
       enabled: true
-      additionalLabels: {{- toYaml .Values.monitoring.labels | nindent 8 }}
+      additionalLabels: {{- include "common.tplvalues.render" (dict "value" .Values.monitoring.labels "context" .) | nindent 8 }}
     {{- end }}
     deschedulerPolicy: {{- $telemetryConf := include "common.telemetry.conf" (dict "protocol" "otlp") | fromYaml }}
       {{- if and $telemetryConf.enabled .Values.monitoring.prometheus.enabled }}

charts/base-cluster/templates/dns/external-dns.yaml

@@ -31,6 +31,10 @@ spec:
     - name: kube-prometheus-stack
       namespace: monitoring
   {{- end }}
+  install:
+    crds: Skip
+  upgrade:
+    crds: Skip
   values:
     priorityClassName: cluster-components
     {{- if .Values.global.imageRegistry }}
@@ -47,6 +51,11 @@ spec:
     {{- end }}
     sources:
       - ingress
+      - gateway-httproute
+      - gateway-grpcroute
+      - gateway-tlsroute
+      - gateway-tcproute
+      - gateway-udproute
     rbac:
       create: true
     crd:
@@ -71,5 +80,5 @@ spec:
       enabled: {{ .Values.monitoring.prometheus.enabled }}
       serviceMonitor:
         enabled: {{ .Values.monitoring.prometheus.enabled }}
-        labels: {{- toYaml .Values.monitoring.labels | nindent 10 }}
+        labels: {{- include "common.tplvalues.render" (dict "value" .Values.monitoring.labels "context" .) | nindent 10 }}
 {{- end -}}

charts/base-cluster/templates/flux/podMonitor.yaml

@@ -9,7 +9,7 @@ metadata:
   name: flux
   namespace: {{ .Release.Namespace }}
   labels: {{- include "common.labels.standard" . | nindent 4 }}
-    {{- with .Values.monitoring.labels }}{{- toYaml . | nindent 4 }}{{- end }}
+    {{- with .Values.monitoring.labels }}{{- include "common.tplvalues.render" (dict "value" . "context" $) | nindent 4 }}{{- end }}
     app.kubernetes.io/component: prometheus
     app.kubernetes.io/part-of: flux
 spec:

charts/base-cluster/templates/flux/rules/flux-status.yaml

@@ -9,7 +9,7 @@ metadata:
   name: flux-status
   namespace: {{ .Release.Namespace }}
   labels: {{- include "common.labels.standard" . | nindent 4 }}
-    {{- with .Values.monitoring.labels }}{{- toYaml . | nindent 4 }}{{- end }}
+    {{- with .Values.monitoring.labels }}{{- include "common.tplvalues.render" (dict "value" . "context" $) | nindent 4 }}{{- end }}
     app.kubernetes.io/component: prometheus
     app.kubernetes.io/part-of: flux
 spec:

charts/base-cluster/templates/global/cluster-ingress.yaml

@@ -1,4 +1,4 @@
-{{- if and .Values.dns.provider .Values.global.baseDomain .Values.ingress.enabled }}
+{{- if and .Values.dns.provider .Values.global.baseDomain (ne .Values.ingress.provider "none") }}
 {{- if false }}
 apiVersion: networking.k8s.io/v1
 {{- else }}

charts/base-cluster/templates/ingress/gateway-api.yaml

@@ -0,0 +1,33 @@
+{{- if eq .Values.ingress.provider "traefik" -}}
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: GitRepository
+metadata:
+  name: gateway-api
+  namespace: ingress
+  labels: {{- include "common.labels.standard" $ | nindent 4 }}
+    app.kubernetes.io/component: gateway-api
+spec:
+  interval: 1h
+  url: https://github.com/kubernetes-sigs/gateway-api
+  ref:
+    semver: "1.2.1"
+  ignore: |
+    *
+    !config/crd/experimental
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: gateway-api
+  namespace: ingress
+  labels: {{- include "common.labels.standard" $ | nindent 4 }}
+    app.kubernetes.io/component: gateway-api
+spec:
+  interval: 1h
+  path: config/crd/experimental
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: gateway-api
+    namespace: ingress
+{{- end -}}

charts/base-cluster/templates/ingress/nginx.yaml

@@ -1,4 +1,4 @@
-{{ if .Values.ingress.enabled }}
+{{- if eq .Values.ingress.provider "nginx" -}}
 apiVersion: helm.toolkit.fluxcd.io/v2
 kind: HelmRelease
 metadata:
@@ -26,7 +26,7 @@ spec:
         enabled: {{ .Values.monitoring.prometheus.enabled }}
         serviceMonitor:
           enabled: {{ .Values.monitoring.prometheus.enabled }}
-          additionalLabels: {{- toYaml .Values.monitoring.labels | nindent 12 }}
+          additionalLabels: {{- include "common.tplvalues.render" (dict "value" .Values.monitoring.labels "context" .) | nindent 12 }}
       {{- $telemetryConf := include "common.telemetry.conf" (dict "protocol" "otlp") | fromYaml }}
       {{- if and $telemetryConf.enabled .Values.monitoring.prometheus.enabled }}
       opentelemetry:
@@ -94,4 +94,4 @@ spec:
       image:
         registry: {{ .Values.global.imageRegistry }}
       {{- end }}
-  {{ end }}
+{{- end -}}

charts/base-cluster/templates/ingress/traefik.yaml

@@ -0,0 +1,89 @@
+{{- if eq .Values.ingress.provider "traefik" -}}
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: ingress-controller
+  namespace: ingress
+  labels: {{- include "common.labels.standard" $ | nindent 4 }}
+    app.kubernetes.io/component: ingress
+spec:
+  chart:
+    spec: {{- include "base-cluster.helm.chartSpec" (dict "repo" "traefik" "chart" "traefik" "context" $) | nindent 6 }}
+  interval: 1h
+  driftDetection:
+    mode: enabled
+  {{- if .Values.monitoring.prometheus.enabled }}
+  dependsOn:
+    - name: kube-prometheus-stack
+      namespace: monitoring
+  {{- end }}
+  install:
+    crds: Skip
+  upgrade:
+    crds: Skip
+  values:
+    fullnameOverride: ingress-controller
+    {{- with .Values.global.imageRegistry }}
+    image:
+      registry: {{ . }}
+    {{- end }}
+    deployment:
+      replicas: {{ .Values.ingress.replicas }}
+    ports:
+      web:
+        redirections:
+          entryPoint:
+            to: websecure
+            scheme: https
+            permanent: true
+        proxyProtocol:
+          insecure: {{ .Values.ingress.useProxyProtocol }}
+      websecure:
+        proxyProtocol:
+          insecure: {{ .Values.ingress.useProxyProtocol }}
+    service:
+      annotations:
+        loadbalancer.openstack.org/proxy-protocol: {{ .Values.ingress.useProxyProtocol | quote }}
+        load-balancer.hetzner.cloud/uses-proxyprotocol: {{ .Values.ingress.useProxyProtocol | quote }}
+        load-balancer.hetzner.cloud/disable-private-ingress: "true"
+        {{- if .Values.ingress.IP }}
+        loadbalancer.openstack.org/keep-floatingip: "true"
+        {{- end }}
+      {{- if .Values.ingress.IP }}
+      spec:
+        loadBalancerIP: {{ .Values.ingress.IP | quote }}
+      {{- end }}
+    gatewayClass:
+      name: default
+    gateway:
+      enabled: false # Gateways need to be created for each set of hostnames, therefore having a clusterwide one makes no sense
+    resources: {{- include "common.resources" .Values.ingress | nindent 8 }}
+    metrics:
+      enabled: {{ .Values.monitoring.prometheus.enabled }}
+      serviceMonitor:
+        enabled: {{ .Values.monitoring.prometheus.enabled }}
+        additionalLabels: {{- include "common.tplvalues.render" (dict "value" .Values.monitoring.labels "context" .) | nindent 12 }}
+    providers:
+      kubernetesCRD:
+        enabled: false
+      kubernetesGateway:
+        enabled: true
+        experimentalChannel: true
+    logs:
+      general:
+        format: json
+    globalArguments: # Otherwise these are on by default
+      - --global.sendanonymoususage=false
+      - --global.checknewversion=false
+    {{- $telemetryConf := include "common.telemetry.conf" (dict "protocol" "otlp" "global" .Values.global) | fromYaml }}
+    {{- if $telemetryConf.enabled }}
+    tracing:
+      otlp:
+        enabled: true
+        grpc:
+          endpoint: {{ $telemetryConf.endpoint }}
+          {{- if $telemetryConf.insecure }}
+          insecure: true
+          {{- end }}
+    {{- end }}
+{{- end -}}