From e5230522ea6c68bf459e521ca1be315114c88112 Mon Sep 17 00:00:00 2001 From: Filip Nikolic Date: Thu, 9 May 2024 12:11:48 +0200 Subject: [PATCH] oci-hook: allow users to set namespace exceptions and define default Prior to this commit `kube-system` was always added as an exception in order for Pods in that namespace to be created in the event that the OCI hook was not able to reach the Tetragon agent. This leads to a deadlock scenario when Tetragon itself is not installed in `kube-system` as the agent won't start and leave the cluster in an unhealthy state. Instead of always adding `kube-syste` this change will always add the namespace Tetragon is deployed in, to guarantee that the agent can always start. In addition to that a user can now define additional namespaces as further exceptions. For example to ensure Pods in business-critical namespaces can still be created even if the OCI hook fails to reach the Tetragon agent. Signed-off-by: Filip Nikolic --- contrib/rthooks/tetragon-oci-hook/cmd/setup/main.go | 5 +++-- contrib/rthooks/tetragon-oci-hook/docs/demo.md | 2 +- contrib/rthooks/tetragon-oci-hook/k8s/ds-uninstall.yaml | 1 - docs/content/en/docs/reference/helm-chart.md | 3 ++- install/kubernetes/tetragon/README.md | 3 ++- .../tetragon/templates/_container_oci_hooks_setup.tpl | 1 + install/kubernetes/tetragon/values.yaml | 2 ++ 7 files changed, 11 insertions(+), 6 deletions(-) diff --git a/contrib/rthooks/tetragon-oci-hook/cmd/setup/main.go b/contrib/rthooks/tetragon-oci-hook/cmd/setup/main.go index 9a6efbf4505..6d26ad03187 100644 --- a/contrib/rthooks/tetragon-oci-hook/cmd/setup/main.go +++ b/contrib/rthooks/tetragon-oci-hook/cmd/setup/main.go @@ -28,7 +28,8 @@ type Install struct { HostInstallDir string `required help:"Installation dir (in the host). Used for the binary and the hook logfile."` OciHooks struct { - LocalDir string `default:"/hostHooks" help:"oci-hooks drop-in directory (inside the container)"` + LocalDir string `default:"/hostHooks" help:"oci-hooks drop-in directory (inside the container)"` + FailAllowNamespaces string `help:"Comma-separated list of namespaces to allow Pod creation for, in case tetragon-oci-hook fails to reach Tetragon agent."` } `embed:"" prefix:"oci-hooks."` } @@ -60,7 +61,7 @@ func (i *Install) ociHooksInstall(log *logrus.Logger) { binFname := filepath.Join(i.HostInstallDir, binBaseName) logFname := filepath.Join(i.HostInstallDir, logBaseName) - hook := ociHooksConfig(binFname, "--log-fname", logFname) + hook := ociHooksConfig(binFname, "--log-fname", logFname, "--fail-allow-namespaces", i.OciHooks.FailAllowNamespaces) data, err := json.MarshalIndent(hook, "", " ") if err != nil { log.WithError(err).Fatal("failed to unmarshall hook info") diff --git a/contrib/rthooks/tetragon-oci-hook/docs/demo.md b/contrib/rthooks/tetragon-oci-hook/docs/demo.md index 3f1d4806f2d..161b71e639d 100644 --- a/contrib/rthooks/tetragon-oci-hook/docs/demo.md +++ b/contrib/rthooks/tetragon-oci-hook/docs/demo.md @@ -29,7 +29,7 @@ helm install --namespace kube-system \ --set tetragon.image.override=localhost/cilium/tetragon:latest \ --set tetragon.grpc.address="unix:///var/run/cilium/tetragon/tetragon.sock" \ --set tetragon.ociHookSetup.enabled=true \ - tetragon ./install/kubernetes + tetragon ./install/kubernetes/tetragon ... kubectl logs -n kube-system tetragon-289tf -c oci-hook-setup time="2023-12-05T09:28:50Z" level=info msg="written binary" hook-dst-path=/hostInstall/tetragon-oci-hook diff --git a/contrib/rthooks/tetragon-oci-hook/k8s/ds-uninstall.yaml b/contrib/rthooks/tetragon-oci-hook/k8s/ds-uninstall.yaml index 491f4d00bd4..0817c778764 100644 --- a/contrib/rthooks/tetragon-oci-hook/k8s/ds-uninstall.yaml +++ b/contrib/rthooks/tetragon-oci-hook/k8s/ds-uninstall.yaml @@ -2,7 +2,6 @@ apiVersion: apps/v1 kind: DaemonSet metadata: name: tetragon-oci-hook-uninstall - namespace: kube-system labels: k8s-app: tetragon-oci-hook-setup-test spec: diff --git a/docs/content/en/docs/reference/helm-chart.md b/docs/content/en/docs/reference/helm-chart.md index f51c3c17879..3596061d5fd 100644 --- a/docs/content/en/docs/reference/helm-chart.md +++ b/docs/content/en/docs/reference/helm-chart.md @@ -82,9 +82,10 @@ To use [the values available](#values), with `helm install` or `helm upgrade`, u | tetragon.image.override | string | `nil` | | | tetragon.image.repository | string | `"quay.io/cilium/tetragon"` | | | tetragon.image.tag | string | `"v1.1.0"` | | -| tetragon.ociHookSetup | object | `{"enabled":false,"extraVolumeMounts":[],"installDir":"/opt/tetragon","interface":"oci-hooks","resources":{},"securityContext":{"privileged":true}}` | Configure tetragon's init container for setting up tetragon-oci-hook on the host | +| tetragon.ociHookSetup | object | `{"enabled":false,"extraVolumeMounts":[],"failAllowNamespaces":"","installDir":"/opt/tetragon","interface":"oci-hooks","resources":{},"securityContext":{"privileged":true}}` | Configure tetragon's init container for setting up tetragon-oci-hook on the host | | tetragon.ociHookSetup.enabled | bool | `false` | enable init container to setup tetragon-oci-hook | | tetragon.ociHookSetup.extraVolumeMounts | list | `[]` | Extra volume mounts to add to the oci-hook-setup init container | +| tetragon.ociHookSetup.failAllowNamespaces | string | `""` | Comma-separated list of namespaces to allow Pod creation for, in case tetragon-oci-hook fails to reach Tetragon agent. | | tetragon.ociHookSetup.interface | string | `"oci-hooks"` | interface specifices how the hook is configured. There is only one avaialble value for now: "oci-hooks" (https://github.com/containers/common/blob/main/pkg/hooks/docs/oci-hooks.5.md). | | tetragon.ociHookSetup.resources | object | `{}` | resources for the the oci-hook-setup init container | | tetragon.ociHookSetup.securityContext | object | `{"privileged":true}` | Security context for oci-hook-setup init container | diff --git a/install/kubernetes/tetragon/README.md b/install/kubernetes/tetragon/README.md index 55986abd165..18ca975b458 100644 --- a/install/kubernetes/tetragon/README.md +++ b/install/kubernetes/tetragon/README.md @@ -64,9 +64,10 @@ Helm chart for Tetragon | tetragon.image.override | string | `nil` | | | tetragon.image.repository | string | `"quay.io/cilium/tetragon"` | | | tetragon.image.tag | string | `"v1.1.0"` | | -| tetragon.ociHookSetup | object | `{"enabled":false,"extraVolumeMounts":[],"installDir":"/opt/tetragon","interface":"oci-hooks","resources":{},"securityContext":{"privileged":true}}` | Configure tetragon's init container for setting up tetragon-oci-hook on the host | +| tetragon.ociHookSetup | object | `{"enabled":false,"extraVolumeMounts":[],"failAllowNamespaces":"","installDir":"/opt/tetragon","interface":"oci-hooks","resources":{},"securityContext":{"privileged":true}}` | Configure tetragon's init container for setting up tetragon-oci-hook on the host | | tetragon.ociHookSetup.enabled | bool | `false` | enable init container to setup tetragon-oci-hook | | tetragon.ociHookSetup.extraVolumeMounts | list | `[]` | Extra volume mounts to add to the oci-hook-setup init container | +| tetragon.ociHookSetup.failAllowNamespaces | string | `""` | Comma-separated list of namespaces to allow Pod creation for, in case tetragon-oci-hook fails to reach Tetragon agent. | | tetragon.ociHookSetup.interface | string | `"oci-hooks"` | interface specifices how the hook is configured. There is only one avaialble value for now: "oci-hooks" (https://github.com/containers/common/blob/main/pkg/hooks/docs/oci-hooks.5.md). | | tetragon.ociHookSetup.resources | object | `{}` | resources for the the oci-hook-setup init container | | tetragon.ociHookSetup.securityContext | object | `{"privileged":true}` | Security context for oci-hook-setup init container | diff --git a/install/kubernetes/tetragon/templates/_container_oci_hooks_setup.tpl b/install/kubernetes/tetragon/templates/_container_oci_hooks_setup.tpl index 895277d9bbe..23f8d47ac94 100644 --- a/install/kubernetes/tetragon/templates/_container_oci_hooks_setup.tpl +++ b/install/kubernetes/tetragon/templates/_container_oci_hooks_setup.tpl @@ -11,6 +11,7 @@ - --local-install-dir={{ include "container.tetragonOCIHookSetup.installPath" . }} - --host-install-dir={{ .Values.tetragon.ociHookSetup.installDir }} - --oci-hooks.local-dir={{ include "container.tetragonOCIHookSetup.hooksPath" . }} + - --oci-hooks.fail-allow-namespaces={{ if .Values.tetragon.ociHookSetup.failAllowNamespaces }}{{ printf "%s,%s" .Release.Namespace .Values.tetragon.ociHookSetup.failAllowNamespaces }}{{ else }}{{ .Release.Namespace }}{{ end }} volumeMounts: {{- with .Values.tetragon.ociHookSetup.extraVolumeMounts }} {{- toYaml . | nindent 4 }} diff --git a/install/kubernetes/tetragon/values.yaml b/install/kubernetes/tetragon/values.yaml index d8a8a4c5bfb..0bc6292c5d7 100644 --- a/install/kubernetes/tetragon/values.yaml +++ b/install/kubernetes/tetragon/values.yaml @@ -200,6 +200,8 @@ tetragon: # "oci-hooks" (https://github.com/containers/common/blob/main/pkg/hooks/docs/oci-hooks.5.md). interface: "oci-hooks" installDir: "/opt/tetragon" + # -- Comma-separated list of namespaces to allow Pod creation for, in case tetragon-oci-hook fails to reach Tetragon agent. + failAllowNamespaces: "" # -- Security context for oci-hook-setup init container securityContext: privileged: true