feat: mount odigos agent directory via pod manifest (#2398)

This is preparation for removing the instrumentation device which
currently is responsible for the mount

api/k8sconsts/agent.go

@@ -0,0 +1,6 @@
+package k8sconsts
+
+const (
+	OdigosAgentsDirectory      = "/var/odigos"
+	OdigosAgentMountVolumeName = "odigos-agent"
+)

api/odigos/v1alpha1/instrumentationconfig_types.go

@@ -179,6 +179,15 @@ type InstrumentationConfigSpec struct {
 	SdkConfigs []SdkConfig `json:"sdkConfigs,omitempty"`
 }
 
+func (in *InstrumentationConfigSpec) GetContainerAgentConfig(containerName string) *ContainerAgentConfig {
+	for _, containerConfig := range in.Containers {
+		if containerConfig.ContainerName == containerName {
+			return &containerConfig
+		}
+	}
+	return nil
+}
+
 type SdkConfig struct {
 
 	// The language of the SDK being configured

cli/cmd/resources/odiglet.go

@@ -360,7 +360,7 @@ func NewOdigletDaemonSet(ns string, version string, imagePrefix string, imageNam
 							Name: "odigos",
 							VolumeSource: corev1.VolumeSource{
 								HostPath: &corev1.HostPathVolumeSource{
-									Path: "/var/odigos",
+									Path: k8sconsts.OdigosAgentsDirectory,
 								},
 							},
 						},
@@ -408,7 +408,7 @@ func NewOdigletDaemonSet(ns string, version string, imagePrefix string, imageNam
 							VolumeMounts: append([]corev1.VolumeMount{
 								{
 									Name:      "odigos",
-									MountPath: "/var/odigos",
+									MountPath: k8sconsts.OdigosAgentsDirectory,
 								},
 							}, odigosSeLinuxHostVolumeMounts...),
 							ImagePullPolicy: "IfNotPresent",
@@ -477,7 +477,7 @@ func NewOdigletDaemonSet(ns string, version string, imagePrefix string, imageNam
 								},
 								{
 									Name:      "odigos",
-									MountPath: "/var/odigos",
+									MountPath: k8sconsts.OdigosAgentsDirectory,
 									ReadOnly:  true,
 								},
 								{

distros/distro/oteldistribution.go

@@ -2,6 +2,8 @@ package distro
 
 import "github.com/odigos-io/odigos/common"
 
+const AgentPlaceholderDirectory = "{{ODIGOS_AGENTS_DIR}}"
+
 type RuntimeEnvironment struct {
 	// the runtime environment this distribution targets.
 	// examples: nodejs, JVM, CPython, etc.
@@ -38,6 +40,13 @@ type EnvironmentVariable struct {
 	Delimiter string `yaml:"delimiter"`
 }
 
+type AgentDirectory struct {
+	// The name of a directory where odigos agent files can be found.
+	// The special value {{ODIGOS_AGENTS_DIR}} is replaced with the actual value at this platform.
+	// K8s will mount this directory from the node fs to the container, but other platforms may have different ways for handling this.
+	DirectoryName string `yaml:"directoryName"`
+}
+
 // OtelDistro (Short for OpenTelemetry Distribution) is a collection of OpenTelemetry components,
 // including instrumentations, SDKs, and other components that are distributed together.
 // Each distribution includes a unique name, and metadata about the ways it is implemented.
@@ -78,4 +87,9 @@ type OtelDistro struct {
 	// a list of environment variables that needs to be set in the application runtime
 	// to enable the distribution.
 	EnvironmentVariables []EnvironmentVariable `yaml:"environmentVariables,omitempty"`
+
+	// Directories on the FS where the agent files can be consumed from.
+	// this can be empty if the distribution does not require agent files,
+	// or contain more than one directory if the distribution requires it.
+	AgentDirectories []AgentDirectory `yaml:"agentDirectories,omitempty"`
 }

distros/yamls/dotnet-community.yaml

@@ -19,3 +19,5 @@ spec:
   tiers: 
     - community
     - onprem
+  agentDirectories:
+    - directoryName: "{{ODIGOS_AGENTS_DIR}}/dotnet"

distros/yamls/dotnet-legacy.yaml

@@ -13,3 +13,5 @@ spec:
     This distribution is for Dotnet applications running on old runtimes, using OpenTelemetry Native SDK and instrumentation libraries from the OpenTelemetry community.
   tiers: 
     - onprem
+  agentDirectories:
+    - directoryName: "{{ODIGOS_AGENTS_DIR}}/legacy-dotnet"

distros/yamls/java-community.yaml

@@ -20,4 +20,5 @@ spec:
     - envName: JAVA_TOOL_OPTIONS
       envValue: '-javaagent:{{ODIGOS_AGENTS_DIR}}/java/javaagent.jar'
       delimiter: ' '
-
+  agentDirectories:
+    - directoryName: "{{ODIGOS_AGENTS_DIR}}/java"    

distros/yamls/java-ebpf-instrumentations.yaml

@@ -20,3 +20,5 @@ spec:
     - envName: JAVA_TOOL_OPTIONS
       envValue: '-javaagent:{{ODIGOS_AGENTS_DIR}}/java-ebpf/dtrace-injector.jar'
       delimiter: ' '
+  agentDirectories:
+    - directoryName: "{{ODIGOS_AGENTS_DIR}}/java-ebpf"

distros/yamls/java-enterprise.yaml

@@ -20,3 +20,5 @@ spec:
     - envName: JAVA_TOOL_OPTIONS
       envValue: "-javaagent:{{ODIGOS_AGENTS_DIR}}/java-ext-ebpf/javaagent.jar -Dotel.javaagent.extensions={{ODIGOS_AGENTS_DIR}}/java-ext-ebpf/otel_agent_extension.jar"
       delimiter: ' '
+  agentDirectories:
+    - directoryName: "{{ODIGOS_AGENTS_DIR}}/java-ext-ebpf"

distros/yamls/nodejs-community.yaml

@@ -17,3 +17,5 @@ spec:
     - envName: NODE_OPTIONS
       envValue: '--require {{ODIGOS_AGENTS_DIR}}/nodejs/autoinstrumentation.js'
       delimiter: ' '
+  agentDirectories:
+    - directoryName: "{{ODIGOS_AGENTS_DIR}}/nodejs"

distros/yamls/nodejs-enterprise.yaml

@@ -17,3 +17,5 @@ spec:
     - envName: NODE_OPTIONS
       envValue: '--require {{ODIGOS_AGENTS_DIR}}/nodejs-ebpf/autoinstrumentation.js'
       delimiter: ' '
+  agentDirectories:
+    - directoryName: "{{ODIGOS_AGENTS_DIR}}/nodejs-ebpf"

distros/yamls/python-community.yaml

@@ -17,3 +17,5 @@ spec:
     - envName: PYTHONPATH
       envValue: '{{ODIGOS_AGENTS_DIR}}/python:{{ODIGOS_AGENTS_DIR}}/python/opentelemetry/instrumentation/auto_instrumentation'
       delimiter: ':'
+  agentDirectories:
+    - directoryName: "{{ODIGOS_AGENTS_DIR}}/python"

distros/yamls/python-enterprise.yaml

@@ -17,4 +17,5 @@ spec:
     - envName: PYTHONPATH
       envValue: '{{ODIGOS_AGENTS_DIR}}/python-ebpf:{{ODIGOS_AGENTS_DIR}}/python/opentelemetry/instrumentation/auto_instrumentation:{{ODIGOS_AGENTS_DIR}}/python'
       delimiter: ':'
-
+  agentDirectories:
+    - directoryName: "{{ODIGOS_AGENTS_DIR}}/python-ebpf"

instrumentor/controllers/agentenabled/pods_webhook.go

@@ -10,6 +10,8 @@ import (
 	"github.com/odigos-io/odigos/api/k8sconsts"
 	odigosv1 "github.com/odigos-io/odigos/api/odigos/v1alpha1"
 	"github.com/odigos-io/odigos/common"
+	"github.com/odigos-io/odigos/distros"
+	"github.com/odigos-io/odigos/distros/distro"
 	"github.com/odigos-io/odigos/instrumentor/controllers/utils"
 	podutils "github.com/odigos-io/odigos/instrumentor/internal/pod"
 	webhookdeviceinjector "github.com/odigos-io/odigos/instrumentor/internal/webhook_device_injector"
@@ -78,16 +80,38 @@ func (p *PodsWebhook) Default(ctx context.Context, obj runtime.Object) error {
 		return nil
 	}
 
+	// Add odiglet installed node-affinity to the pod
+	podutils.AddOdigletInstalledAffinity(pod)
+
+	volumeMounted := false
+	for i := range pod.Spec.Containers {
+		podContainerSpec := &pod.Spec.Containers[i]
+		containerConfig := ic.Spec.GetContainerAgentConfig(podContainerSpec.Name)
+		if containerConfig == nil {
+			// no config is found for this container, so skip (don't inject anything to it)
+			continue
+		}
+
+		containerVolumeMounted, err := injectOdigosToContainer(containerConfig, podContainerSpec)
+		if err != nil {
+			logger.Error(err, "failed to inject ODIGOS agent to container")
+			continue
+		}
+		volumeMounted = volumeMounted || containerVolumeMounted
+	}
+
+	if volumeMounted {
+		// only mount the volume if at least one container has a volume to mount
+		mountPodVolume(pod)
+	}
+
 	// Inject ODIGOS environment variables and instrumentation device into all containers
 	injectErr := p.injectOdigosInstrumentation(ctx, pod, &ic, pw)
 	if injectErr != nil {
 		logger.Error(injectErr, "failed to inject ODIGOS instrumentation. Skipping Injection of ODIGOS agent")
 		return nil
 	}
 
-	// Add odiglet installed node-affinity to the pod
-	podutils.AddOdigletInstalledAffinity(pod)
-
 	return nil
 }
 
@@ -191,6 +215,48 @@ func (p *PodsWebhook) injectOdigosInstrumentation(ctx context.Context, pod *core
 	return nil
 }
 
+func mountDirectory(containerSpec *corev1.Container, dir string) {
+	// TODO: assuming the directory always starts with {{ODIGOS_AGENTS_DIR}}. This should be validated.
+	// Should we return errors here to validate static values?
+	relativePath := strings.TrimPrefix(dir, distro.AgentPlaceholderDirectory+"/")
+	absolutePath := strings.ReplaceAll(dir, distro.AgentPlaceholderDirectory, k8sconsts.OdigosAgentsDirectory)
+	containerSpec.VolumeMounts = append(containerSpec.VolumeMounts, corev1.VolumeMount{
+		Name:      k8sconsts.OdigosAgentMountVolumeName,
+		SubPath:   relativePath,
+		MountPath: absolutePath,
+		ReadOnly:  true,
+	})
+}
+
+func injectOdigosToContainer(containerConfig *odigosv1.ContainerAgentConfig, podContainerSpec *corev1.Container) (bool, error) {
+
+	distroName := containerConfig.OtelDistroName
+
+	distroMetadata := distros.GetDistroByName(distroName)
+	if distroMetadata == nil {
+		return false, fmt.Errorf("distribution %s not found", distroName)
+	}
+
+	volumeMounted := false
+	for _, agentDirectory := range distroMetadata.AgentDirectories {
+		mountDirectory(podContainerSpec, agentDirectory.DirectoryName)
+		volumeMounted = true
+	}
+
+	return volumeMounted, nil
+}
+
+func mountPodVolume(pod *corev1.Pod) {
+	pod.Spec.Volumes = append(pod.Spec.Volumes, corev1.Volume{
+		Name: k8sconsts.OdigosAgentMountVolumeName,
+		VolumeSource: corev1.VolumeSource{
+			HostPath: &corev1.HostPathVolumeSource{
+				Path: k8sconsts.OdigosAgentsDirectory,
+			},
+		},
+	})
+}
+
 func envVarsExist(containerEnv []corev1.EnvVar, commonEnvVars []corev1.EnvVar) bool {
 	envMap := make(map[string]struct{})
 	for _, envVar := range containerEnv {

instrumentor/runtimemigration/runtimemigration.go

@@ -247,7 +247,7 @@ func handleContainerRuntimeDetailsUpdate(
 				continue
 			}
 
-			if strings.Contains(*envValue, "/var/odigos") {
+			if strings.Contains(*envValue, k8sconsts.OdigosAgentsDirectory) {
 				cleanedEnvValue := envOverwrite.CleanupEnvValueFromOdigosAdditions(envKey, *envValue)
 				annotationEnvVarsForContainer[envKey] = &cleanedEnvValue
 				needToUpdateWorkloadAnnotation = true
@@ -276,7 +276,7 @@ func handleContainerRuntimeDetailsUpdate(
 				filteredEnvVars := []v1alpha1.EnvVar{}
 
 				for _, envVar := range containerRuntimeDetails.EnvFromContainerRuntime {
-					if strings.Contains(envVar.Value, "/var/odigos") {
+					if strings.Contains(envVar.Value, k8sconsts.OdigosAgentsDirectory) {
 						// Skip the entry
 						continue
 					}
@@ -378,7 +378,7 @@ func revertOriginalEnvAnnotationInPlace(originalWorkloadEnvVar *envoverwrite.Ori
 		newContainerEnvs := []corev1.EnvVar{}
 		for _, manifestEnvVar := range containerManifest.Env {
 
-			if !strings.Contains(manifestEnvVar.Value, "/var/odigos") {
+			if !strings.Contains(manifestEnvVar.Value, k8sconsts.OdigosAgentsDirectory) {
 				// we only revert values that odigos overwrote,
 				// if the value is not odigos value, keep it as is
 				newContainerEnvs = append(newContainerEnvs, manifestEnvVar)

k8sutils/pkg/cri/criwrapper.go

@@ -14,6 +14,7 @@ import (
 	"google.golang.org/grpc/credentials/insecure"
 	criapi "k8s.io/cri-api/pkg/apis/runtime/v1"
 
+	"github.com/odigos-io/odigos/api/k8sconsts"
 	odigosv1 "github.com/odigos-io/odigos/api/odigos/v1alpha1"
 )
 
@@ -157,7 +158,7 @@ func (rc *CriClient) GetContainerEnvVarsList(ctx context.Context, envVarKeys []s
 			// If the environment variable originates from the device, it will still be observed in the CRI.
 			// In this case, it should not be set as envFromContainerRuntime.
 			// We can be certain that it is not coming from the manifest, as CRI is only queried when the variable is not found in the manifest.
-			if strings.Contains(value, "/var/odigos") {
+			if strings.Contains(value, k8sconsts.OdigosAgentsDirectory) {
 				continue
 			}
 			result = append(result, odigosv1.EnvVar{Name: key, Value: value})

odiglet/pkg/instrumentation/fs/agents.go

@@ -10,12 +10,12 @@ import (
 	"strings"
 	"syscall"
 
+	"github.com/odigos-io/odigos/api/k8sconsts"
 	"github.com/odigos-io/odigos/odiglet/pkg/log"
 )
 
 const (
 	containerDir   = "/instrumentations"
-	hostDir        = "/var/odigos"
 	chrootDir      = "/host"
 	semanagePath   = "/sbin/semanage"
 	restoreconPath = "/sbin/restorecon"
@@ -38,18 +38,18 @@ func CopyAgentsDirectoryToHost() error {
 		"/var/odigos/python-ebpf/pythonUSDT.abi3.so":                                                   {},
 	}
 
-	updatedFilesToKeepMap, err := removeChangedFilesFromKeepMap(filesToKeep, containerDir, hostDir)
+	updatedFilesToKeepMap, err := removeChangedFilesFromKeepMap(filesToKeep, containerDir, k8sconsts.OdigosAgentsDirectory)
 	if err != nil {
 		log.Logger.Error(err, "Error getting changed files")
 	}
 
-	err = removeFilesInDir(hostDir, updatedFilesToKeepMap)
+	err = removeFilesInDir(k8sconsts.OdigosAgentsDirectory, updatedFilesToKeepMap)
 	if err != nil {
 		log.Logger.Error(err, "Error removing instrumentation directory from host")
 		return err
 	}
 
-	err = copyDirectories(containerDir, hostDir, updatedFilesToKeepMap)
+	err = copyDirectories(containerDir, k8sconsts.OdigosAgentsDirectory, updatedFilesToKeepMap)
 	if err != nil {
 		log.Logger.Error(err, "Error copying instrumentation directory to host")
 		return err
@@ -73,7 +73,7 @@ func CopyAgentsDirectoryToHost() error {
 		_, err = exec.LookPath(restoreconPath)
 		if err == nil {
 			// Run the restorecon command to apply the new context
-			cmd := exec.Command(restoreconPath, "-r", "/var/odigos")
+			cmd := exec.Command(restoreconPath, "-r", k8sconsts.OdigosAgentsDirectory)
 			err = cmd.Run()
 			if err != nil {
 				log.Logger.Error(err, "Error running restorecon command")