CI: Improved K8s Kubeconformance validation

.github/workflows/lint-helm.yaml

@@ -2,8 +2,8 @@ name: Lint helm chart
 on:
   push:
     branches:
-      - main
-      - v*
+    - main
+    - v*
     paths:
     - 'install/kubernetes/**'
     - 'pkg/k8s/apis/cilium.io/client/crds/v1alpha1/*.yaml'
@@ -14,12 +14,65 @@ on:
     - 'pkg/k8s/apis/cilium.io/client/crds/v1alpha1/*.yaml'
     - '.github/workflows/lint-helm.yaml'
 
+env:
+  MIN_K8S_VERSION: "1.23.0"
+  # renovate: datasource=python-version
+  PYTHON_VERSION: "3.12"
+
 jobs:
   generated-files:
     runs-on: ubuntu-latest
     steps:
+      # Get source
       - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
-      - name: Run install/kubernetes
+
+      # Check / install dependencies
+      - name: Check if vendored openapi2jsonschema.py script is up to date
+        run: |
+          make -C install/kubernetes openapi2jsonschema.py
+          test -z "$(git status --porcelain)"
+          if [ $? != 0 ]; then
+            git status --porcelain
+            echo "Vendored openapi2jsonschema.py script is out of date."
+            echo "Please run 'make -C install/kubernetes openapi2jsonschema.py' and submit your changes."; exit 1
+          fi
+      - name: Setup Python
+        id: setup-python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+      - name: Install pipenv
+        id: install-pipenv
+        shell: bash
+        run: |
+          python -m pip install --upgrade --no-cache-dir pip
+          python -m pip install --no-cache-dir pipenv
+      - name: Cache Pipfile
+        id: cache-pipfile
+        uses: actions/cache@v4
+        with:
+          path: ~/.local/share/virtualenvs
+          key: ${{ runner.os }}-python-${{ steps.setup-python.outputs.python-version }}-pipenv-${{ hashFiles('Pipfile.lock') }}
+      - name: Sync Pipfile
+        id: sync-pipfile
+        shell: bash
+        working-directory: install/kubernetes
+        run: |
+          pipenv sync --dev --python ${{ env.PYTHON_VERSION }}
+      - name: Check if vendored CRD JSON schemas are up to date
+        run: |
+          make -C install/kubernetes generate-jsonschemas
+          test -z "$(git status --porcelain)"
+          if [ $? != 0 ]; then
+            git status --porcelain
+            echo "Vendored CRD JSON schemas are out of date."
+            echo "Please run 'make -C install/kubernetes generate-jsonschemas' and submit your changes."; exit 1
+          fi
+      - name: Install Helm CLI # Required for the Helm chart templating
+        uses: azure/[email protected]
+
+      # Validate Helm chart
+      - name: Generate Helm chart
         run: |
           make -C install/kubernetes
       - name: Validate generated files
@@ -29,3 +82,46 @@ jobs:
             git status --porcelain
             echo "Please run 'make -C install/kubernetes' and submit your changes."; exit 1
           fi
+
+      # (Re-)run Kubeconform checks explicitly once again to catch specific errors in that regard (to get the STDOUT/STDERR)
+      - name: Run Kubeconform with minimum supported K8s version
+        id: kubeconform_min_k8s_version
+        uses: mathiasvr/[email protected]
+        with:
+          shell: bash
+          run: |
+            make -C install/kubernetes kubeconform K8S_VERSION=${{ env.MIN_K8S_VERSION }}
+      - name: Run Kubeconform with latest K8s version
+        id: kubeconform_latest_k8s_version
+        uses: mathiasvr/[email protected]
+        with:
+          shell: bash
+          run: |
+            make -C install/kubernetes kubeconform
+
+      # Post Kubeconform issues as comment on the GH PR, if there are any
+      - name: Comment Kubeconform Output
+        if: failure() && (steps.kubeconform_min_k8s_version.outcome != 'success' || steps.kubeconform_latest_k8s_version.outcome != 'success')
+        uses: marocchino/sticky-pull-request-comment@v2
+        with:
+          hide_and_recreate: true
+          skip_unchanged: true
+          message: |
+            ## Kubeconform with minimum supported K8s version ${{ env.MIN_K8S_VERSION }}
+            STDOUT:
+            ```
+            ${{ steps.kubeconform_min_k8s_version.outputs.stdout }}
+            ```
+            STDERR:
+            ```
+            ${{ steps.kubeconform_min_k8s_version.outputs.stderr }}
+            ```
+            ## Kubeconform with latest K8s version
+            STDOUT:
+            ```
+            ${{ steps.kubeconform_latest_k8s_version.outputs.stdout }}
+            ```
+            STDERR:
+            ```
+            ${{ steps.kubeconform_latest_k8s_version.outputs.stderr }}
+            ```

install/kubernetes/Makefile

@@ -7,15 +7,21 @@ HELM_IMAGE=docker.io/alpine/helm:3.15.3@sha256:ba0dcbbcf31f780bd8cdeeabc44bc6939
 KUBECONFORM_IMAGE=ghcr.io/yannh/kubeconform:v0.6.7-alpine@sha256:824e0c248809e4b2da2a768b16b107cf17ada88a89ec6aa6050e566ba93ebbc6
 # renovate: datasource=docker
 HELMDOCS_IMAGE=docker.io/jnorwood/helm-docs:v1.14.2@sha256:7e562b49ab6b1dbc50c3da8f2dd6ffa8a5c6bba327b1c6335cc15ce29267979c
+# renovate: datasource=github-releases depName=yannh/kubeconform
+KUBECONFORM_VERSION := v0.6.7
+PYTHON := python3
+PIPENV := pipenv
+K8S_VERSION := master
 
 REPO_ROOT := $(shell git rev-parse --show-toplevel)
 TETRAGON_CHART := tetragon
 CRDS := $(REPO_ROOT)/pkg/k8s/apis/cilium.io/client/crds/v1alpha1
+JSON_SCHEMAS := $(REPO_ROOT)/install/kubernetes/schemas
 
 HELM ?= docker run --rm -v $(CURDIR)/$(TETRAGON_CHART):/apps $(HELM_IMAGE)
 
 .PHONY: all
-all: deps $(TETRAGON_CHART)/crds-yaml lint docs
+all: deps $(TETRAGON_CHART)/crds-yaml lint docs openapi2jsonschema.py generate-jsonschemas kubeconform
 
 .PHONY: deps
 deps: 
@@ -24,7 +30,6 @@ deps:
 .PHONY: lint
 lint:
 	$(HELM) lint . --with-subcharts
-	$(HELM) template tetragon . | docker run --rm -i $(KUBECONFORM_IMAGE) --strict --schema-location default
 
 .PHONY: docs
 docs:
@@ -40,3 +45,35 @@ docs:
 .PHONY: $(TETRAGON_CHART)/crds-yaml
 $(TETRAGON_CHART)/crds-yaml: $(CRDS)
 	cp -rf $(CRDS)/. $(TETRAGON_CHART)/crds-yaml
+
+# openapi2jsonschema.py script generating JSON schema from the CRD YAML spec.
+.PHONY: openapi2jsonschema.py
+openapi2jsonschema.py:
+	curl -sSfLO https://raw.githubusercontent.com/yannh/kubeconform/$(KUBECONFORM_VERSION)/scripts/$@
+
+# To validate (using openapi2jsonschema.py) default Ruleset policies included in the Helm chart, we need to pass the
+# JSON schema of the TracingPolicy CRD. This target generates such schema. It requires pipenv to be pre-installed.
+.PHONY: generate-jsonschemas
+generate-jsonschemas: $(CRDS)
+	mkdir -p $(JSON_SCHEMAS)/
+	pipenv install
+	FILENAME_FORMAT='{kind}-{fullgroup}' $(PIPENV) run $(PYTHON) openapi2jsonschema.py $(CRDS)/*
+	mv $(REPO_ROOT)/install/kubernetes/*-cilium.io.json $(JSON_SCHEMAS)/
+	pipenv --rm
+
+.PHONY: kubeconform
+kubeconform:
+	@echo "## Testing Helm chart: \"$(TETRAGON_CHART)\""
+	$(HELM) template $(TETRAGON_CHART) . \
+	-f values.yaml \
+	--set crds.installMethod=helm \
+	--set tracingPolicies.default.enabled=true |\
+		docker run --rm -i -v $(JSON_SCHEMAS):/schemas $(KUBECONFORM_IMAGE) \
+		-summary \
+		-verbose \
+		-schema-location default \
+		-schema-location '/schemas/{{ .ResourceKind }}-{{ .Group }}.json' \
+		-skip CustomResourceDefinition \
+		-strict \
+		-kubernetes-version $(K8S_VERSION)
+	@echo ""

install/kubernetes/Pipfile

@@ -0,0 +1,12 @@
+[[source]]
+url = "https://pypi.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[packages]
+pyyaml = "*"
+
+[dev-packages]
+
+[requires]
+python_version = "3.12"