Merge pull request #1131 from aiyengar2/fix_crd_chart

Fix CRD chart regression and fix CRDs for upgrades

packages/rancher-monitoring/package.yaml

@@ -2,7 +2,7 @@ url: https://github.com/prometheus-community/helm-charts.git
 subdirectory: charts/kube-prometheus-stack
 commit: 3ca6ba66032a1efce0500f9ad6f83351ad0604b8
 packageVersion: 00
-releaseCandidateVersion: 08
+releaseCandidateVersion: 09
 additionalCharts:
   - workingDir: charts-crd
     crdOptions:

packages/rancher-monitoring/templates/crd-template/README.md

@@ -1,2 +1,24 @@
 # rancher-monitoring-crd
 A Rancher chart that installs the CRDs used by rancher-monitoring.
+
+## How does this chart work?
+
+This chart marshalls all of the CRD files placed in the `crd-manifest` directory into a ConfigMap that is installed onto a cluster alongside relevant RBAC (ServiceAccount, ClusterRoleBinding, ClusterRole, and PodSecurityPolicy).
+
+Once the relevant dependent resourcees are installed / upgraded / rolled back, this chart executes a post-install / post-upgrade / post-rollback Job that:
+- Patches any existing versions of the CRDs contained within the `crd-manifest` on the cluster to set `spec.preserveUnknownFields=false`; this step is required since, based on [Kubernetes docs](https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions/#field-pruning) and a [known workaround](https://github.com/kubernetes-sigs/controller-tools/issues/476#issuecomment-691519936), such CRDs cannot be upgraded normally from `apiextensions.k8s.io/v1beta1` to `apiextensions.k8s.io/v1`.
+- Runs a `kubectl apply` on the CRDs that are contained within the crd-manifest ConfigMap to upgrade CRDs in the cluster
+
+On an uninstall, this chart executes a separate post-delete Job that:
+- Patches any existing versions of the CRDs contained within `crd-manifest` on the cluster to set `metadata.finalizers=[]`
+- Runs a `kubectl delete` on the CRDs that are contained within the crd-manifest ConfigMap to clean up the CRDs from the cluster
+
+Note: If the relevant CRDs already existed in the cluster at the time of install, this chart will absorb ownership of the lifecycle of those CRDs; therefore, on a `helm uninstall`, those CRDs will also be removed from the cluster alongside this chart.
+
+## Why can't we just place the CRDs in the templates/ directory of the main chart?
+
+In Helm today, you cannot declare a CRD and declare a resource of that CRD's kind in templates/ without encountering a failure on render.
+
+## [Helm 3] Why can't we just place the CRDs in the crds/ directory of the main chart?
+
+The Helm 3 `crds/` directory only supports the installation of CRDs, but does not support the upgrade and removal of CRDs, unlike what this chart facilitiates.

packages/rancher-monitoring/templates/crd-template/templates/jobs.yaml

@@ -20,28 +20,28 @@ spec:
         runAsNonRoot: true
         runAsUser: 1000
       initContainers:
-        - name: delete-crds
+        - name: set-preserve-unknown-fields-false
           image: {{ template "system_default_registry" . }}{{ .Values.image.repository }}:{{ .Values.image.tag }}
           imagePullPolicy: IfNotPresent
           command:
-          - /bin/kubectl
-          - delete
-          - --ignore-not-found=true
-          - -f
-          - /etc/config/crd-manifest.yaml
-          volumeMounts:
-            - name: crd-manifest
-              readOnly: true
-              mountPath: /etc/config
+          - /bin/sh
+          - -c
+          - >
+            {{- range $path, $_ :=  (.Files.Glob  "crd-manifest/**.yaml") }}
+            {{- $crd := get (get ($.Files.Get $path | fromYaml) "metadata") "name" }}
+            if [[ -n "$(kubectl get crd {{ $crd }} -o jsonpath='{.spec.preserveUnknownFields}')" ]]; then
+              kubectl patch crd {{ $crd }} -p '{"spec": {"preserveUnknownFields": false, "versions": [{"name": "v1", "served": false, "storage": true}]}}' --type="merge" || true;
+            fi;
+            {{- end }}
       containers:
         - name: create-crds
           image: {{ template "system_default_registry" . }}{{ .Values.image.repository }}:{{ .Values.image.tag }}
           imagePullPolicy: IfNotPresent
           command:
-          - /bin/kubectl
-          - apply
-          - -f
-          - /etc/config/crd-manifest.yaml
+          - /bin/sh
+          - -c
+          - >
+            kubectl apply -f /etc/config/crd-manifest.yaml
           volumeMounts:
             - name: crd-manifest
               readOnly: true
@@ -80,10 +80,13 @@ spec:
           image: {{ template "system_default_registry" . }}{{ .Values.image.repository }}:{{ .Values.image.tag }}
           imagePullPolicy: IfNotPresent
           command:
-          - /bin/kubectl
-          - apply
-          - -f
-          - /etc/config/crd-manifest.yaml
+          - /bin/sh
+          - -c
+          - >
+            {{- range $path, $_ :=  (.Files.Glob  "crd-manifest/**.yaml") }}
+            {{- $crd := get (get ($.Files.Get $path | fromYaml) "metadata") "name" }}
+            kubectl patch crd {{ $crd }} -p '{"metadata": {"finalizers": []}}' || true;
+            {{- end }}
           volumeMounts:
             - name: crd-manifest
               readOnly: true
@@ -93,10 +96,10 @@ spec:
           image: {{ template "system_default_registry" . }}{{ .Values.image.repository }}:{{ .Values.image.tag }}
           imagePullPolicy: IfNotPresent
           command:
-          - /bin/kubectl
-          - delete
-          - -f
-          - /etc/config/crd-manifest.yaml
+          - /bin/sh
+          - -c
+          - >
+            kubectl delete -f /etc/config/crd-manifest.yaml
           volumeMounts:
             - name: crd-manifest
               readOnly: true

packages/rancher-monitoring/templates/crd-template/values.yaml

@@ -7,5 +7,5 @@ global:
     systemDefaultRegistry: ""
 
 image:
-  repository: rancher/kubectl
-  tag: v1.20.2
+  repository: rancher/rancher-agent
+  tag: v2.5.7