feat: add config option for mount method hostpath or virtual device (#…

…2459) This allows the user to choose how to apply the mount. - `k8s-host-path` will add volume to the pod with "host-path". - `k8s-virtual-device` will add the "instrumentation.odigos.io/generic" device to the resource part of relevant containers. This allows control of how the mounting is achieved. Future work: auto detect if hostpath may fail and automatically fallback to virtual device. --------- Co-authored-by: Tamir David <[email protected]>
odigos-io · Feb 17, 2025 · 442f266 · 442f266
1 parent 90bf334
commit 442f266
Show file tree

Hide file tree

Showing 17 changed files with 263 additions and 27 deletions.
diff --git a/api/k8sconsts/device.go b/api/k8sconsts/device.go
@@ -0,0 +1,9 @@
+package k8sconsts
+
+const (
+	// the name of the device that only mounts the odigos agents root directory,
+	// allowing any agent to be access it's files.
+	// it would be more ideal to only mount what is needed,
+	// but it's not desirable to have tons of different devices for each case.
+	OdigosGenericDeviceName = "instrumentation.odigos.io/generic"
+)
diff --git a/cli/cmd/config.go b/cli/cmd/config.go
@@ -11,6 +11,7 @@ import (
 	"github.com/odigos-io/odigos/cli/pkg/log"
 	"github.com/odigos-io/odigos/common"
 	"github.com/odigos-io/odigos/common/consts"
+	"github.com/odigos-io/odigos/k8sutils/pkg/getters"
 	"github.com/spf13/cobra"
 )
 
@@ -29,6 +30,7 @@ var configCmd = &cobra.Command{
 	- "ui-mode": Sets the UI mode(normal/readonly).
 	- "ignored-namespaces": List of namespaces to be ignored.
 	- "ignored-containers": List of containers to be ignored.
+	- "mount-method": Determines how Odigos agent files are mounted into the pod's container filesystem. Options include k8s-host-path (direct hostPath mount) and k8s-virtual-device (virtual device-based injection).
 	`,
 }
 
@@ -70,7 +72,13 @@ var setConfigCmd = &cobra.Command{
 			os.Exit(1)
 		}
 
-		resourceManagers := resources.CreateResourceManagers(client, ns, currentTier, nil, config, "Updating Config")
+		currentOdigosVersion, err := getters.GetOdigosVersionInClusterFromConfigMap(ctx, client.Clientset, ns)
+		if err != nil {
+			fmt.Println("Odigos config failed - unable to read the current Odigos version.")
+			os.Exit(1)
+		}
+
+		resourceManagers := resources.CreateResourceManagers(client, ns, currentTier, nil, config, currentOdigosVersion)
 		err = resources.ApplyResourceManagers(ctx, client, resourceManagers, "Updating Config")
 		if err != nil {
 			l.Error(fmt.Errorf("failed to apply updated configuration: %w", err))
@@ -147,6 +155,19 @@ func setConfigProperty(config *common.OdigosConfiguration, property string, valu
 		}
 		config.IgnoredContainers = value
 
+	case "mount-method":
+		if len(value) != 1 {
+			return fmt.Errorf("%s expects exactly one value", property)
+		}
+		mountMethod := common.MountMethod(value[0])
+		switch mountMethod {
+		case common.K8sHostPathMountMethod:
+		case common.K8sVirtualDeviceMountMethod:
+		default:
+			return fmt.Errorf("invalid mount method: %s (valid values: %s, %s)", value[0], common.K8sHostPathMountMethod, common.K8sVirtualDeviceMountMethod)
+		}
+		config.MountMethod = &mountMethod
+
 	default:
 		return fmt.Errorf("invalid property: %s", property)
 	}

diff --git a/common/mount.go b/common/mount.go
@@ -0,0 +1,14 @@
+package common
+
+// Note: this configuration is currently only relevant for k8s,
+// but is used in odigosconfig which is declared in the common package.
+// We should revisit this decision later on and consider if the config should be k8s specific,
+// then move it to the api module.
+
+// +kubebuilder:validation:Enum=virtual-device;host-path
+type MountMethod string
+
+const (
+	K8sVirtualDeviceMountMethod MountMethod = "k8s-virtual-device"
+	K8sHostPathMountMethod      MountMethod = "k8s-host-path"
+)
diff --git a/common/odigos_config.go b/common/odigos_config.go
@@ -112,4 +112,5 @@ type OdigosConfiguration struct {
 	AllowConcurrentAgents     *bool                          `json:"allowConcurrentAgents,omitempty"`
 	UiMode                    UiMode                         `json:"uiMode,omitempty"`
 	CentralBackendURL         string                         `json:"centralBackendURL,omitempty"`
+	MountMethod               *MountMethod                   `json:"mountMethod,omitempty"`
 }
diff --git a/distros/yamls/java-ebpf-instrumentations.yaml b/distros/yamls/java-ebpf-instrumentations.yaml
@@ -20,5 +20,5 @@ spec:
       delimiter: ' '
   runtimeAgent:
     directoryNames:
-      # - "{{ODIGOS_AGENTS_DIR}}/java-ebpf"
+      - "{{ODIGOS_AGENTS_DIR}}/java-ebpf"
     device: 'instrumentation.odigos.io/generic'
diff --git a/distros/yamls/nodejs-enterprise.yaml b/distros/yamls/nodejs-enterprise.yaml
@@ -17,5 +17,5 @@ spec:
       delimiter: ' '
   runtimeAgent:
     directoryNames:
-      # - "{{ODIGOS_AGENTS_DIR}}/nodejs-ebpf"
+      - "{{ODIGOS_AGENTS_DIR}}/nodejs-ebpf"
     device: 'instrumentation.odigos.io/generic'
diff --git a/docs/cli/odigos_config.mdx b/docs/cli/odigos_config.mdx
@@ -20,6 +20,7 @@ Manage Odigos configuration settings to customize system behavior.
 	- "ui-mode": Sets the UI mode(normal/readonly).
 	- "ignored-namespaces": List of namespaces to be ignored.
 	- "ignored-containers": List of containers to be ignored.
+	- "mount-method": Determines how Odigos agent files are mounted into the pod's container filesystem. Options include k8s-host-path (direct hostPath mount) and k8s-virtual-device (virtual device-based injection).
 
 
 ### Options

diff --git a/docs/instrumentations/configuration/mount-method.mdx b/docs/instrumentations/configuration/mount-method.mdx
@@ -0,0 +1,93 @@
+---
+title: "Agent Mount Method"
+sidebarTitle: "Mount Method"
+icon: "gear"
+description: "For Odigos agents to run inside instrumented pod containers, certain files must be mounted into the container. In Kubernetes, these files are mounted under `/var/odigos` directory, with subdirectories for each agent. There are few mechanisms to achieve this, which are explained in this document."
+---
+
+## Mount Method
+
+<Warning>
+This section is for advanced users and odigos administrators.
+
+It is recommended to use the default settings, unless you have specific requirements or run into an issue.
+</Warning>
+
+## Supported Mount Methods
+
+Odigos supports 2 mount methods, which can be used depending on the user preference, cluster policies, integration with existing tools, etc.
+
+### 1. VirtualDevice
+
+<Info>
+This is the default mount method
+</Info>
+
+#### Pod Manifest Additions
+
+For agents and languages that requires filesystem mounts, odigos webhook will add the following:
+
+- For each instrumented container in the pod spec, odigos will add a resource requirement under the `spec.containers[].resources` field:
+
+```yaml
+    resources:
+      limits:
+        instrumentation.odigos.io/generic: "1"
+      requests:
+        instrumentation.odigos.io/generic: "1"
+```
+
+#### Caveats
+
+- May sometimes not integrate well with node auto-scaling tools like [Karpenter](https://karpenter.sh/).
+- Odiglet daemonset needs to be running on a node for instrumented pods to be scheduled on that node.
+
+### 2. HostPath
+
+This method is an opt-in configuration option which can be used if the virtual device method is not suitable for your cluster.
+
+If it is supported in the cluster, it is preferred to use over the VirtualDevice method which requires odiglet component to run.
+
+**Enabling HostPath**
+
+- Profile: `odigos profile add mount-method-k8s-host-path`
+
+- Odigos CLI: `odigos config set mount-method k8s-host-path`
+
+- Helm Chart: in your values file, set `instrumentor.mountMethod` to `k8s-host-path`, or use helm cli `--set instrumentor.mountMethod=k8s-host-path` flag with helm upgrade.
+
+- Kubernetes Manifest: under the `odigos-config` ConfigMap in odigos namespace, set the value in the `mountMethod` field of `config.yaml` to `k8s-host-path`.
+
+**Pod Manifest Additions**
+
+For agents and languages that requires filesystem mounts, odigos webhook will add the following:
+
+- Volume to the pod spec, under the `spec.volumes` field:
+
+```yaml
+  - hostPath:
+      path: /var/odigos
+      type: ""
+    name: odigos-agent
+```
+
+- VolumeMount to the instrumented container specs, under the `spec.containers[].volumeMounts` field:
+
+```yaml
+    - mountPath: /var/odigos/{agent_sub_dir}
+      name: odigos-agent
+      readOnly: true
+      subPath: {agent_sub_dir}
+```
+
+**Caveats**
+
+The "HostPath" option should be enabled in the cluster. Some policy tools may block this, like: 
+
+- [Open Policy Agent](https://www.openpolicyagent.org/)
+- [Kyverno](https://kyverno.io/)
+
+If your cluster enforces such policies, you have the following options:
+
+- Use the VirtualDevice method.
+- Request the cluster administrator to whitelist the Odigos agent host path in the policy tool.
diff --git a/docs/mint.json b/docs/mint.json
@@ -145,6 +145,13 @@
             "instrumentations/dotnet/native",
             "instrumentations/dotnet/enrichment"
           ]
+        },
+        {
+          "group": "Advanced",
+          "icon": "gear",
+          "pages": [
+            "instrumentations/configuration/mount-method"
+          ]
         }
       ]
     },

diff --git a/helm/odigos/templates/odigos-config-cm.yaml b/helm/odigos/templates/odigos-config-cm.yaml
@@ -89,3 +89,9 @@ data:
       {{- toYaml .Values.ignoredNamespaces | nindent 8 }}
     ignoredContainers:
       {{- toYaml .Values.ignoredContainers | nindent 8 }}
+    {{- if and .Values.instrumentor.mountMethod (has .Values.instrumentor.mountMethod (list "k8s-host-path" "k8s-virtual-device")) }}
+    mountMethod: {{ .Values.instrumentor.mountMethod }}
+    {{- else }}
+    {{- fail "Error: Invalid mountMethod. Supported values are 'k8s-host-path' and 'k8s-virtual-device'." }}
+    {{- end }}
+    
diff --git a/helm/odigos/values.yaml b/helm/odigos/values.yaml
@@ -127,6 +127,10 @@ ui:
   centralBackendURL: ''
 
 instrumentor:
+  # which mount method to use for odigos agent directory
+  # k8s-virtual-device: default method using a virtual device
+  # k8s-host-path: alternative which uses hostPath volume (recommended if supported, requires hostPath volume to be enabled in the cluster)
+  mountMethod: 'k8s-virtual-device'
   deleteLangDetectionPods: true
   image:
     repository: keyval/odigos-instrumentor

diff --git a/instrumentor/controllers/agentenabled/pods_webhook.go b/instrumentor/controllers/agentenabled/pods_webhook.go
@@ -17,6 +17,7 @@ import (
 	webhookenvinjector "github.com/odigos-io/odigos/instrumentor/internal/webhook_env_injector"
 	"github.com/odigos-io/odigos/instrumentor/sdks"
 	sourceutils "github.com/odigos-io/odigos/k8sutils/pkg/source"
+	k8sutils "github.com/odigos-io/odigos/k8sutils/pkg/utils"
 	"github.com/odigos-io/odigos/k8sutils/pkg/workload"
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
@@ -70,6 +71,17 @@ func (p *PodsWebhook) Default(ctx context.Context, obj runtime.Object) error {
 		return nil
 	}
 
+	odigosConfig, err := k8sutils.GetCurrentOdigosConfig(ctx, p.Client)
+	if err != nil {
+		logger.Error(err, "failed to get ODIGOS config. Skipping Injection of ODIGOS agent")
+		return nil
+	}
+	if odigosConfig.MountMethod == nil {
+		// we are reading the effective config which should already have the mount method resolved or defaulted
+		logger.Error(errors.New("mount method is not set in ODIGOS config"), "Skipping Injection of ODIGOS agent")
+		return nil
+	}
+
 	// this is temporary and should be refactored so the service name and other resource attributes are written to agent config
 	serviceName := p.getServiceNameForEnv(ctx, logger, pw)
 	if serviceName == nil || *serviceName == "" {
@@ -94,15 +106,15 @@ func (p *PodsWebhook) Default(ctx context.Context, obj runtime.Object) error {
 			continue
 		}
 
-		containerVolumeMounted, err := p.injectOdigosToContainer(containerConfig, podContainerSpec, *pw, *serviceName)
+		containerVolumeMounted, err := p.injectOdigosToContainer(containerConfig, podContainerSpec, *pw, *serviceName, *odigosConfig.MountMethod)
 		if err != nil {
 			logger.Error(err, "failed to inject ODIGOS agent to container")
 			continue
 		}
 		volumeMounted = volumeMounted || containerVolumeMounted
 	}
 
-	if volumeMounted {
+	if *odigosConfig.MountMethod == common.K8sHostPathMountMethod && volumeMounted {
 		// only mount the volume if at least one container has a volume to mount
 		podswebhook.MountPodVolume(pod)
 	}
@@ -180,7 +192,7 @@ func (p *PodsWebhook) injectOdigosInstrumentation(ctx context.Context, pod *core
 	return nil
 }
 
-func (p *PodsWebhook) injectOdigosToContainer(containerConfig *odigosv1.ContainerAgentConfig, podContainerSpec *corev1.Container, pw k8sconsts.PodWorkload, serviceName string) (bool, error) {
+func (p *PodsWebhook) injectOdigosToContainer(containerConfig *odigosv1.ContainerAgentConfig, podContainerSpec *corev1.Container, pw k8sconsts.PodWorkload, serviceName string, mountMethod common.MountMethod) (bool, error) {
 
 	distroName := containerConfig.OtelDistroName
 
@@ -197,30 +209,40 @@ func (p *PodsWebhook) injectOdigosToContainer(containerConfig *odigosv1.Containe
 
 	volumeMounted := false
 	if distroMetadata.RuntimeAgent != nil {
-		for _, agentDirectoryName := range distroMetadata.RuntimeAgent.DirectoryNames {
-			podswebhook.MountDirectory(podContainerSpec, agentDirectoryName)
-			volumeMounted = true
+		if mountMethod == common.K8sHostPathMountMethod {
+			// mount directory only if the mount type is host-path
+			for _, agentDirectoryName := range distroMetadata.RuntimeAgent.DirectoryNames {
+				podswebhook.MountDirectory(podContainerSpec, agentDirectoryName)
+				volumeMounted = true
+			}
 		}
 		if distroMetadata.RuntimeAgent.K8sAttrsViaEnvVars {
 			podswebhook.InjectOtelResourceAndServerNameEnvVars(&existingEnvNames, podContainerSpec, distroName, pw, serviceName)
 		}
 		// TODO: once we have a flag to enable/disable device injection, we should check it here.
 		if distroMetadata.RuntimeAgent.Device != nil {
-			deviceName := *distroMetadata.RuntimeAgent.Device
-			// TODO: currently devices are composed with glibc as input for dotnet.
-			// as devices will soon converge to a single device, I am hardcoding the logic here,
-			// which will eventually be removed once dotnet specific devices are removed.
-			if containerConfig.DistroParams != nil {
-				libcType, ok := containerConfig.DistroParams[common.LibcTypeDistroParameterName]
-				if ok {
-					libcPrefix := ""
-					if libcType == string(common.Musl) {
-						libcPrefix = "musl-"
+
+			// amir 17 feb 2025, this is here only for migration.
+			// even if mount method is not device, we still need to inject the deprecated agent specific device
+			// while we remove them one by one
+			isGenericDevice := *distroMetadata.RuntimeAgent.Device == k8sconsts.OdigosGenericDeviceName
+			if mountMethod == common.K8sVirtualDeviceMountMethod || !isGenericDevice {
+				deviceName := *distroMetadata.RuntimeAgent.Device
+				// TODO: currently devices are composed with glibc as input for dotnet.
+				// as devices will soon converge to a single device, I am hardcoding the logic here,
+				// which will eventually be removed once dotnet specific devices are removed.
+				if containerConfig.DistroParams != nil {
+					libcType, ok := containerConfig.DistroParams[common.LibcTypeDistroParameterName]
+					if ok {
+						libcPrefix := ""
+						if libcType == string(common.Musl) {
+							libcPrefix = "musl-"
+						}
+						deviceName = strings.ReplaceAll(deviceName, "{{param.LIBC_TYPE}}", libcPrefix)
 					}
-					deviceName = strings.ReplaceAll(deviceName, "{{param.LIBC_TYPE}}", libcPrefix)
 				}
+				podswebhook.InjectDeviceToContainer(podContainerSpec, deviceName)
 			}
-			podswebhook.InjectDeviceToContainer(podContainerSpec, deviceName)
 		}
 	}
 	podswebhook.InjectOdigosK8sEnvVars(&existingEnvNames, podContainerSpec, distroName, pw.Namespace)

diff --git a/profiles/aggregators/greatwall.go b/profiles/aggregators/greatwall.go
@@ -9,9 +9,10 @@ var GreatwallProfile = profile.Profile{
 	ProfileName: common.ProfileName("greatwall"),
 	MinimumTier: common.OnPremOdigosTier,
 	ShortDescription: "Bundle profile that includes " +
-		"java-ebpf-instrumentations and legacy-dotnet-instrumentation",
+		"specific preset for on-premises installations.",
 	Dependencies: []common.ProfileName{
 		"java-ebpf-instrumentations",
 		"legacy-dotnet-instrumentation",
+		"mount-method-k8s-virtual-device",
 	},
 }
diff --git a/profiles/aggregators/kratos.go b/profiles/aggregators/kratos.go
@@ -9,10 +9,7 @@ var KratosProfile = profile.Profile{
 	ProfileName: common.ProfileName("kratos"),
 	MinimumTier: common.OnPremOdigosTier,
 	ShortDescription: "Bundle profile that includes " +
-		"db-payload-collection, semconv, category-attributes, copy-scope, " +
-		"hostname-as-podname, code-attributes, query-operation-detector, " +
-		"disableNameProcessorProfile, small-batches, size_m, " +
-		"allow_concurrent_agents",
+		"specific presets for on-premises installations.",
 	Dependencies: []common.ProfileName{
 		"db-payload-collection",
 		"semconv",
@@ -24,5 +21,7 @@ var KratosProfile = profile.Profile{
 		"disable-name-processor",
 		"small-batches",
 		"size_m",
-		"allow_concurrent_agents"},
+		"allow_concurrent_agents",
+		"mount-method-k8s-host-path",
+	},
 }
diff --git a/profiles/allprofiles.go b/profiles/allprofiles.go
@@ -28,6 +28,8 @@ var AllProfiles = []profile.Profile{
 	instrumentation.JavaEbpfInstrumentationsProfile,
 	instrumentation.JavaNativeInstrumentationsProfile,
 	instrumentation.LegacyDotNetProfile,
+	instrumentation.MountMethodK8sHostPathProfile,
+	instrumentation.MountMethodK8sVirtualDevice,
 
 	pipeline.DisableNameProcessorProfile,
 	pipeline.SmallBatchesProfile,