ingress-nginx production example

This patch adds a production addon values file with best practices for using ingress-nginx. Signed-off-by: Kevin Fox <[email protected]>
spiffe · Apr 3, 2023 · 8eb37f6 · 8eb37f6
1 parent 1b4bfb7
commit 8eb37f6
Show file tree

Hide file tree

Showing 8 changed files with 184 additions and 4 deletions.
diff --git a/.github/tests/production-ingress-nginx/install.sh b/.github/tests/production-ingress-nginx/install.sh
@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+
+set -e
+
+kubectl create namespace spire-system
+kubectl label namespace spire-system pod-security.kubernetes.io/enforce=privileged
+kubectl create namespace spire-server
+kubectl label namespace spire-server pod-security.kubernetes.io/enforce=restricted
+
+helm install cert-manager cert-manager --namespace cert-manager --create-namespace --version v1.11.0 --set installCRDs=true --repo https://charts.jetstack.io --wait
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+kubectl apply -f $SCRIPT_DIR/testcert.yaml -n spire-server
+
+helm install ingress-nginx ingress-nginx --version 4.5.2 --repo https://kubernetes.github.io/ingress-nginx --create-namespace -n ingress-nginx --wait \
+    --set controller.extraArgs.enable-ssl-passthrough=,controller.admissionWebhooks.enabled=false,controller.service.type=ClusterIP \
+    --set controller.ingressClassResource.default=true
+
+ip=$(kubectl get svc -n ingress-nginx ingress-nginx-controller -o go-template='{{ .spec.clusterIP }}')
+echo $ip oidc-discovery.example.org
+
+cat > /tmp/dummydns <<EOF
+spiffe-oidc-discovery-provider:
+  tests:
+    hostAliases:
+    - ip: "$ip"
+      hostnames:
+      - "oidc-discovery.example.org"
+EOF
+
+helm upgrade --install --namespace spire-server spire charts/spire -f examples/production/values.yaml -f examples/production/values-export-ingress-nginx.yaml \
+    -f /tmp/dummydns --set spiffe-oidc-discovery-provider.tests.tls.customCA=tls-cert --wait
diff --git a/.github/tests/production-ingress-nginx/post-install.sh b/.github/tests/production-ingress-nginx/post-install.sh
@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+
+set -x
+
+SCRIPT=$(readlink -f "$0")
+SCRIPTPATH=$(dirname "$SCRIPT")
+
+k_wait=(kubectl wait --for condition=available --timeout 30s --namespace)
+k_rollout_status=(kubectl rollout status --watch --timeout 30s --namespace)
+
+cat <<EOF >>"$GITHUB_STEP_SUMMARY"
+### spire
+| workload | Status |
+| -------- | ------ |
+| spire-server | $("${k_rollout_status[@]}" spire-server statefulset spire-server) |
+| spire-spiffe-oidc-discovery-provider | $("${k_wait[@]}" spire-server deployments.apps spire-spiffe-oidc-discovery-provider) |
+| spire-spiffe-csi-driver | $("${k_rollout_status[@]}" spire-system daemonset spire-spiffe-csi-driver) |
+| spire-agent | $("${k_rollout_status[@]}" spire-system daemonset spire-agent) |
+EOF
+
+if [ $1 -ne 0 ]; then
+  for ns in spire-server spire-system ingress-nginx cert-manager; do
+    echo
+    echo '```'
+    echo "==> Events of namespace $ns"
+    echo "........................................................................................................................"
+    echo ">>> kubectl --request-timeout=30s get events --output wide --namespace $ns"
+    kubectl --request-timeout=30s get events --output wide --namespace $ns
+    echo "........................................................................................................................"
+    echo "<== Events of namespace $ns"
+    echo "........................................................................................................................"
+    echo ">>> kubectl --request-timeout=30s describe pods --namespace $ns"
+    kubectl --request-timeout=30s describe pods --namespace $ns
+    echo "========================================================================================================================"
+    kubectl get pods -o name -n $ns | while read line; do echo logs for $line; kubectl logs -n $ns $line --all-containers=true --ignore-errors=true; done
+    echo '========================================================================================================================'
+    echo '```'
+  done
+fi | cat >> "$GITHUB_STEP_SUMMARY"
diff --git a/.github/tests/production-ingress-nginx/testcert.yaml b/.github/tests/production-ingress-nginx/testcert.yaml
@@ -0,0 +1,44 @@
+---
+apiVersion: cert-manager.io/v1
+kind: Issuer
+metadata:
+  name: selfsigned-issuer
+spec:
+  selfSigned: {}
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: demo-selfsigned-ca
+spec:
+  isCA: true
+  commonName: demo-selfsigned-ca
+  secretName: root-secret
+  privateKey:
+    algorithm: ECDSA
+    size: 256
+  issuerRef:
+    name: selfsigned-issuer
+    kind: Issuer
+    group: cert-manager.io
+---
+apiVersion: cert-manager.io/v1
+kind: Issuer
+metadata:
+  name: demo-ca
+spec:
+  ca:
+    secretName: root-secret
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: oidc
+spec:
+  dnsNames:
+  - oidc-discovery.example.org
+  secretName: tls-cert
+  issuerRef:
+    name: selfsigned-issuer
+    kind: Issuer
+    group: cert-manager.io
diff --git a/charts/spire/charts/spiffe-oidc-discovery-provider/README.md b/charts/spire/charts/spiffe-oidc-discovery-provider/README.md
@@ -25,8 +25,8 @@ A Helm chart to install the SPIFFE OIDC discovery provider.
 | config.acme.directoryUrl | string | `"https://acme-v02.api.letsencrypt.org/directory"` |  |
 | config.acme.emailAddress | string | `"[email protected]"` |  |
 | config.acme.tosAccepted | bool | `false` |  |
-| config.domains[0] | string | `"localhost"` |  |
-| config.domains[1] | string | `"oidc-discovery.example.org"` |  |
+| config.domains[0] | string | `"oidc-discovery.example.org"` |  |
+| config.domains[1] | string | `"localhost"` |  |
 | config.logLevel | string | `"info"` |  |
 | fullnameOverride | string | `""` |  |
 | image.pullPolicy | string | `"IfNotPresent"` |  |
@@ -68,6 +68,8 @@ A Helm chart to install the SPIFFE OIDC discovery provider.
 | telemetry.prometheus.nginxExporter.image.version | string | `"0.11.0"` |  |
 | telemetry.prometheus.nginxExporter.resources | object | `{}` |  |
 | telemetry.prometheus.port | int | `9988` |  |
+| tests.hostAliases | list | `[]` |  |
+| tests.tls.customCA | string | `""` |  |
 | tolerations | list | `[]` |  |
 | trustDomain | string | `"example.org"` |  |
 

diff --git a/charts/spire/charts/spiffe-oidc-discovery-provider/templates/tests/test-connection.yaml b/charts/spire/charts/spiffe-oidc-discovery-provider/templates/tests/test-connection.yaml
@@ -30,11 +30,36 @@ spec:
       securityContext:
         {{- toYaml .Values.securityContext | nindent 8 }}
     {{- if .Values.ingress.enabled }}
-    - name: wget-ingress
+    - name: ingress
+      {{- if .Values.tests.tls.enabled }}
+      image: cgr.dev/chainguard/bash:latest
+      command: ['curl']
+      {{- if .Values.tests.tls.customCA }}
+      args: ['--cacert', '/ca/ca.crt', 'https://{{ index .Values.config.domains 0 }}/.well-known/openid-configuration']
+      {{- else }}
+      args: ['https://{{ index .Values.config.domains 0 }}/.well-known/openid-configuration']
+      {{- end }}
+      {{- if .Values.tests.tls.customCA }}
+      volumeMounts:
+      - name: custom-ca
+        mountPoint: /ca
+      {{- end }}
+      {{- else }}
       image: cgr.dev/chainguard/bash:latest
       command: ['wget']
       args: ['-O', '/dev/null', '{{ index .Values.config.domains 0 }}/.well-known/openid-configuration']
+      {{- end }}
+      {{- if ne (len .Values.tests.hostAliases) 0 }}
+      hostAliases:
+        {{- toYaml .Values.tests.hostAliases | nindent 8 }}
+      {{- end }}
       securityContext:
         {{- toYaml .Values.securityContext | nindent 8 }}
     {{- end }}
+  {{- if .Values.tests.tls.customCA }}
+  volumes:
+  - name: custom-ca
+    secret:
+      secretName: {{ .Values.tests.tls.customCA }}
+  {{- end }}
   restartPolicy: Never
diff --git a/charts/spire/charts/spiffe-oidc-discovery-provider/values.yaml b/charts/spire/charts/spiffe-oidc-discovery-provider/values.yaml
@@ -77,8 +77,8 @@ insecureScheme:
 config:
   logLevel: info
   domains:
-    - localhost
     - oidc-discovery.example.org
+    - localhost
 
   acme:
     tosAccepted: false
@@ -155,3 +155,8 @@ ingress:
   #  - secretName: chart-example-tls
   #    hosts:
   #      - oidc-discovery.example.org
+
+tests:
+  hostAliases: []
+  tls:
+    customCA: ""
diff --git a/examples/production/README.md b/examples/production/README.md
@@ -8,7 +8,14 @@ kubectl label namespace "spire-system" pod-security.kubernetes.io/enforce=privil
 kubectl create namespace "spire-server"
 kubectl label namespace "spire-server" pod-security.kubernetes.io/enforce=restricted
 
+```
 helm upgrade --install --namespace spire-server spire charts/spire -f values.yaml
 ```
 
+If your using ingress-nginx and want to expose spire outside the cluster, add the
+following to the end of the helm upgrade example:
+```
+-f values-export-ingress-nginx.yaml
+```
+
 See [values.yaml](./values.yaml) for more details on the chart configurations to achieve this setup.
diff --git a/examples/production/values-export-ingress-nginx.yaml b/examples/production/values-export-ingress-nginx.yaml
@@ -0,0 +1,27 @@
+spiffe-oidc-discovery-provider:
+  tests:
+    tls:
+      enabled: true
+  ingress:
+    enabled: true
+    annotations:
+      nginx.ingress.kubernetes.io/ssl-redirect: "true"
+      nginx.ingress.kubernetes.io/force-ssl-redirect: "true"
+
+# Optional settings you may put in your own values.
+#      kubernetes.io/ingress.class: nginx
+#      kubernetes.io/tls-acme: "true"
+#
+#    className: nginx
+
+# You must override these in your own values file with the appropriate hostname and secret
+# or it wont start.
+    hosts:
+      - host: oidc-discovery.example.org
+        paths:
+          - path: /
+            pathType: Prefix
+    tls:
+      - secretName: tls-cert
+        hosts:
+          - oidc-discovery.example.org