refactor: migrate otel resource env to distro (#2407)

This PR migrate the code in webhook that injects OTEL_RESOURCE_ATTRIBUTE
and OTEL_SERVICE_NAME to the 3 relevant distributions.

It moves code from the main pods_webhook.go file to internal files for
better organization

Co-authored-by: Ben Elferink <[email protected]>

distros/distro/oteldistribution.go

@@ -40,11 +40,18 @@ type EnvironmentVariable struct {
 	Delimiter string `yaml:"delimiter"`
 }
 
-type AgentDirectory struct {
+type RuntimeAgent struct {
 	// The name of a directory where odigos agent files can be found.
 	// The special value {{ODIGOS_AGENTS_DIR}} is replaced with the actual value at this platform.
-	// K8s will mount this directory from the node fs to the container, but other platforms may have different ways for handling this.
-	DirectoryName string `yaml:"directoryName"`
+	// K8s will mount this directory from the node fs to the container. other platforms may have different ways to make the directory accessible.
+	DirectoryNames []string `yaml:"directoryNames"`
+
+	// This field indicates that the agent populates k8s resource attributes via environment variables.
+	// It targets distros odigos uses without wrapper or customization.
+	// For eBPF, the resource attributes are set in code.
+	// For opamp distros, the resource attributes are set in the opamp server.
+	// We will eventually remove this field once all distros upgrade to dynamic resource attributes.
+	K8sAttrsViaEnvVars bool `yaml:"k8sAttrsViaEnvVars,omitempty"`
 }
 
 // OtelDistro (Short for OpenTelemetry Distribution) is a collection of OpenTelemetry components,
@@ -88,8 +95,7 @@ type OtelDistro struct {
 	// to enable the distribution.
 	EnvironmentVariables []EnvironmentVariable `yaml:"environmentVariables,omitempty"`
 
-	// Directories on the FS where the agent files can be consumed from.
-	// this can be empty if the distribution does not require agent files,
-	// or contain more than one directory if the distribution requires it.
-	AgentDirectories []AgentDirectory `yaml:"agentDirectories,omitempty"`
+	// Metadata and properties of the runtime agent that is used to enable the distribution.
+	// Can be nil in case no runtime agent is required.
+	RuntimeAgent *RuntimeAgent `yaml:"runtimeAgent,omitempty"`
 }

distros/yamls/dotnet-community.yaml

@@ -19,5 +19,7 @@ spec:
   tiers: 
     - community
     - onprem
-  agentDirectories:
-    - directoryName: "{{ODIGOS_AGENTS_DIR}}/dotnet"
+  runtimeAgent:
+    directoryNames:
+      - "{{ODIGOS_AGENTS_DIR}}/dotnet"
+    k8sAttrsViaEnvVars: true

distros/yamls/dotnet-legacy.yaml

@@ -13,5 +13,7 @@ spec:
     This distribution is for Dotnet applications running on old runtimes, using OpenTelemetry Native SDK and instrumentation libraries from the OpenTelemetry community.
   tiers: 
     - onprem
-  agentDirectories:
-    - directoryName: "{{ODIGOS_AGENTS_DIR}}/legacy-dotnet"
+  runtimeAgent:
+    directoryNames:
+      - "{{ODIGOS_AGENTS_DIR}}/legacy-dotnet"
+    k8sAttrsViaEnvVars: true

distros/yamls/java-community.yaml

@@ -20,5 +20,7 @@ spec:
     - envName: JAVA_TOOL_OPTIONS
       envValue: '-javaagent:{{ODIGOS_AGENTS_DIR}}/java/javaagent.jar'
       delimiter: ' '
-  agentDirectories:
-    - directoryName: "{{ODIGOS_AGENTS_DIR}}/java"    
+  runtimeAgent:
+    directoryNames:
+      - "{{ODIGOS_AGENTS_DIR}}/java"
+    k8sAttrsViaEnvVars: true

distros/yamls/java-ebpf-instrumentations.yaml

@@ -20,5 +20,6 @@ spec:
     - envName: JAVA_TOOL_OPTIONS
       envValue: '-javaagent:{{ODIGOS_AGENTS_DIR}}/java-ebpf/dtrace-injector.jar'
       delimiter: ' '
-  agentDirectories:
-    - directoryName: "{{ODIGOS_AGENTS_DIR}}/java-ebpf"
+  runtimeAgent:
+    directoryNames:
+      - "{{ODIGOS_AGENTS_DIR}}/java-ebpf"

distros/yamls/java-enterprise.yaml

@@ -20,5 +20,6 @@ spec:
     - envName: JAVA_TOOL_OPTIONS
       envValue: "-javaagent:{{ODIGOS_AGENTS_DIR}}/java-ext-ebpf/javaagent.jar -Dotel.javaagent.extensions={{ODIGOS_AGENTS_DIR}}/java-ext-ebpf/otel_agent_extension.jar"
       delimiter: ' '
-  agentDirectories:
-    - directoryName: "{{ODIGOS_AGENTS_DIR}}/java-ext-ebpf"
+  runtimeAgent:
+    directoryNames:
+      - "{{ODIGOS_AGENTS_DIR}}/java-ext-ebpf"

distros/yamls/nodejs-community.yaml

@@ -17,5 +17,6 @@ spec:
     - envName: NODE_OPTIONS
       envValue: '--require {{ODIGOS_AGENTS_DIR}}/nodejs/autoinstrumentation.js'
       delimiter: ' '
-  agentDirectories:
-    - directoryName: "{{ODIGOS_AGENTS_DIR}}/nodejs"
+  runtimeAgent:
+    directoryNames:
+      - "{{ODIGOS_AGENTS_DIR}}/nodejs"

distros/yamls/nodejs-enterprise.yaml

@@ -17,5 +17,6 @@ spec:
     - envName: NODE_OPTIONS
       envValue: '--require {{ODIGOS_AGENTS_DIR}}/nodejs-ebpf/autoinstrumentation.js'
       delimiter: ' '
-  agentDirectories:
-    - directoryName: "{{ODIGOS_AGENTS_DIR}}/nodejs-ebpf"
+  runtimeAgent:
+    directoryNames:
+      - "{{ODIGOS_AGENTS_DIR}}/nodejs-ebpf"

distros/yamls/python-community.yaml

@@ -17,5 +17,7 @@ spec:
     - envName: PYTHONPATH
       envValue: '{{ODIGOS_AGENTS_DIR}}/python:{{ODIGOS_AGENTS_DIR}}/python/opentelemetry/instrumentation/auto_instrumentation'
       delimiter: ':'
-  agentDirectories:
-    - directoryName: "{{ODIGOS_AGENTS_DIR}}/python"
+  runtimeAgent:
+    directoryNames:
+      - "{{ODIGOS_AGENTS_DIR}}/python"
+

distros/yamls/python-enterprise.yaml

@@ -17,6 +17,7 @@ spec:
     - envName: PYTHONPATH
       envValue: '{{ODIGOS_AGENTS_DIR}}/python-ebpf:{{ODIGOS_AGENTS_DIR}}/python/opentelemetry/instrumentation/auto_instrumentation:{{ODIGOS_AGENTS_DIR}}/python'
       delimiter: ':'
-  agentDirectories:
-    - directoryName: "{{ODIGOS_AGENTS_DIR}}/python-ebpf"
-    - directoryName: "{{ODIGOS_AGENTS_DIR}}/python"
+  runtimeAgent:
+    directoryNames:
+      - "{{ODIGOS_AGENTS_DIR}}/python"
+      - "{{ODIGOS_AGENTS_DIR}}/python-ebpf"

instrumentor/controllers/agentenabled/pods_webhook.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"strings"
 
 	"github.com/go-logr/logr"
 	"github.com/odigos-io/odigos/api/k8sconsts"
@@ -19,8 +18,6 @@ import (
 	"github.com/odigos-io/odigos/instrumentor/sdks"
 	sourceutils "github.com/odigos-io/odigos/k8sutils/pkg/source"
 	"github.com/odigos-io/odigos/k8sutils/pkg/workload"
-	"go.opentelemetry.io/otel/attribute"
-	semconv "go.opentelemetry.io/otel/semconv/v1.26.0"
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -30,14 +27,6 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
 )
 
-const otelServiceNameEnvVarName = "OTEL_SERVICE_NAME"
-const otelResourceAttributesEnvVarName = "OTEL_RESOURCE_ATTRIBUTES"
-
-type resourceAttribute struct {
-	Key   attribute.Key
-	Value string
-}
-
 type PodsWebhook struct {
 	client.Client
 }
@@ -80,6 +69,13 @@ func (p *PodsWebhook) Default(ctx context.Context, obj runtime.Object) error {
 		return nil
 	}
 
+	// this is temporary and should be refactored so the service name and other resource attributes are written to agent config
+	serviceName := p.getServiceNameForEnv(ctx, logger, pw)
+	if serviceName == nil || *serviceName == "" {
+		logger.Error(errors.New("failed to get service name for pod"), "Skipping Injection of ODIGOS agent")
+		return nil
+	}
+
 	// Add odiglet installed node-affinity to the pod
 	podutils.AddOdigletInstalledAffinity(pod)
 
@@ -97,7 +93,7 @@ func (p *PodsWebhook) Default(ctx context.Context, obj runtime.Object) error {
 			continue
 		}
 
-		containerVolumeMounted, err := injectOdigosToContainer(containerConfig, podContainerSpec, *pw)
+		containerVolumeMounted, err := injectOdigosToContainer(containerConfig, podContainerSpec, *pw, *serviceName)
 		if err != nil {
 			logger.Error(err, "failed to inject ODIGOS agent to container")
 			continue
@@ -159,9 +155,6 @@ func (p *PodsWebhook) injectOdigosInstrumentation(ctx context.Context, pod *core
 		return fmt.Errorf("failed to determine OpenTelemetry SDKs: %w", err)
 	}
 
-	var serviceName *string
-	var serviceNameEnv *corev1.EnvVar
-
 	for i := range pod.Spec.Containers {
 		container := &pod.Spec.Containers[i]
 		runtimeDetails := ic.Status.GetRuntimeDetailsForContainer(*container)
@@ -189,37 +182,11 @@ func (p *PodsWebhook) injectOdigosInstrumentation(ctx context.Context, pod *core
 		}
 
 		webhookenvinjector.InjectOdigosAgentEnvVars(logger, *pw, container, otelSdk, runtimeDetails)
-
-		if shouldInjectServiceName(runtimeDetails.Language, otelSdk) {
-			// Ensure the serviceName is fetched only once per pod
-			if serviceName == nil {
-				serviceName = p.getServiceNameForEnv(ctx, logger, pw)
-			}
-			// Initialize serviceNameEnv only once per pod if serviceName is valid
-			if serviceName != nil && serviceNameEnv == nil {
-				serviceNameEnv = &corev1.EnvVar{
-					Name:  otelServiceNameEnvVarName,
-					Value: *serviceName,
-				}
-			}
-
-			if !otelNameExists(container.Env) {
-				container.Env = append(container.Env, *serviceNameEnv)
-			}
-		}
-
-		resourceAttributes := getResourceAttributes(pw, container.Name)
-		resourceAttributesEnvValue := getResourceAttributesEnvVarValue(resourceAttributes)
-
-		container.Env = append(container.Env, corev1.EnvVar{
-			Name:  otelResourceAttributesEnvVarName,
-			Value: resourceAttributesEnvValue,
-		})
 	}
 	return nil
 }
 
-func injectOdigosToContainer(containerConfig *odigosv1.ContainerAgentConfig, podContainerSpec *corev1.Container, pw k8sconsts.PodWorkload) (bool, error) {
+func injectOdigosToContainer(containerConfig *odigosv1.ContainerAgentConfig, podContainerSpec *corev1.Container, pw k8sconsts.PodWorkload, serviceName string) (bool, error) {
 
 	distroName := containerConfig.OtelDistroName
 
@@ -228,79 +195,25 @@ func injectOdigosToContainer(containerConfig *odigosv1.ContainerAgentConfig, pod
 		return false, fmt.Errorf("distribution %s not found", distroName)
 	}
 
-	volumeMounted := false
-	for _, agentDirectory := range distroMetadata.AgentDirectories {
-		podswebhook.MountDirectory(podContainerSpec, agentDirectory.DirectoryName)
-		volumeMounted = true
-	}
-	podswebhook.InjectOdigosK8sEnvVars(podContainerSpec, distroName, pw.Namespace)
-
-	return volumeMounted, nil
-}
-
-func getWorkloadKindAttributeKey(podWorkload *k8sconsts.PodWorkload) attribute.Key {
-	switch podWorkload.Kind {
-	case k8sconsts.WorkloadKindDeployment:
-		return semconv.K8SDeploymentNameKey
-	case k8sconsts.WorkloadKindStatefulSet:
-		return semconv.K8SStatefulSetNameKey
-	case k8sconsts.WorkloadKindDaemonSet:
-		return semconv.K8SDaemonSetNameKey
-	}
-	return attribute.Key("")
-}
-
-func getResourceAttributes(podWorkload *k8sconsts.PodWorkload, containerName string) []resourceAttribute {
-	if podWorkload == nil {
-		return []resourceAttribute{}
-	}
-
-	workloadKindKey := getWorkloadKindAttributeKey(podWorkload)
-	return []resourceAttribute{
-		{
-			Key:   semconv.K8SContainerNameKey,
-			Value: containerName,
-		},
-		{
-			Key:   semconv.K8SNamespaceNameKey,
-			Value: podWorkload.Namespace,
-		},
-		{
-			Key:   workloadKindKey,
-			Value: podWorkload.Name,
-		},
-	}
-}
-
-func getResourceAttributesEnvVarValue(ra []resourceAttribute) string {
-	var attrs []string
-	for _, a := range ra {
-		attrs = append(attrs, fmt.Sprintf("%s=%s", a.Key, a.Value))
+	// check for existing env vars so we don't introduce them again
+	existingEnvNames := make(map[string]struct{})
+	for _, envVar := range podContainerSpec.Env {
+		existingEnvNames[envVar.Name] = struct{}{}
 	}
-	return strings.Join(attrs, ",")
-}
 
-func otelNameExists(containerEnv []corev1.EnvVar) bool {
-	for _, envVar := range containerEnv {
-		if envVar.Name == otelServiceNameEnvVarName {
-			return true
+	volumeMounted := false
+	if distroMetadata.RuntimeAgent != nil {
+		for _, agentDirectoryName := range distroMetadata.RuntimeAgent.DirectoryNames {
+			podswebhook.MountDirectory(podContainerSpec, agentDirectoryName)
+			volumeMounted = true
+		}
+		if distroMetadata.RuntimeAgent.K8sAttrsViaEnvVars {
+			podswebhook.InjectOtelResourceAndServerNameEnvVars(&existingEnvNames, podContainerSpec, distroName, pw, serviceName)
 		}
 	}
-	return false
-}
+	podswebhook.InjectOdigosK8sEnvVars(&existingEnvNames, podContainerSpec, distroName, pw.Namespace)
 
-// this is used to set the OTEL_SERVICE_NAME for programming languages and otel sdks that requires it.
-// eBPF instrumentations sets the service name in code, thus it's not needed here.
-// OpAMP sends the service name in the protocol, thus it's not needed here.
-// We are only left with OSS Java and Dotnet that requires the OTEL_SERVICE_NAME to be set.
-func shouldInjectServiceName(pl common.ProgrammingLanguage, otelsdk common.OtelSdk) bool {
-	if pl == common.DotNetProgrammingLanguage {
-		return true
-	}
-	if pl == common.JavaProgrammingLanguage && otelsdk.SdkTier == common.CommunityOtelSdkTier {
-		return true
-	}
-	return false
+	return volumeMounted, nil
 }
 
 // checks for the service name on the annotation, or fallback to the workload name

instrumentor/controllers/agentenabled/podswebhook/env.go

@@ -5,18 +5,12 @@ import (
 	corev1 "k8s.io/api/core/v1"
 )
 
-func InjectOdigosK8sEnvVars(container *corev1.Container, distroName string, ns string) {
+func InjectOdigosK8sEnvVars(existingEnvNames *map[string]struct{}, container *corev1.Container, distroName string, ns string) {
 
-	// check for existing env vars so we don't introduce them again
-	existingEnvNames := make(map[string]struct{})
-	for _, envVar := range container.Env {
-		existingEnvNames[envVar.Name] = struct{}{}
-	}
-
-	injectEnvVarToPodContainer(&existingEnvNames, container, k8sconsts.OdigosEnvVarContainerName, container.Name)
-	injectEnvVarToPodContainer(&existingEnvNames, container, k8sconsts.OdigosEnvVarDistroName, distroName)
-	injectEnvVarObjectFieldRefToPodContainer(&existingEnvNames, container, k8sconsts.OdigosEnvVarPodName, "metadata.name")
-	injectEnvVarToPodContainer(&existingEnvNames, container, k8sconsts.OdigosEnvVarNamespace, ns)
+	injectEnvVarToPodContainer(existingEnvNames, container, k8sconsts.OdigosEnvVarContainerName, container.Name)
+	injectEnvVarToPodContainer(existingEnvNames, container, k8sconsts.OdigosEnvVarDistroName, distroName)
+	injectEnvVarObjectFieldRefToPodContainer(existingEnvNames, container, k8sconsts.OdigosEnvVarPodName, "metadata.name")
+	injectEnvVarToPodContainer(existingEnvNames, container, k8sconsts.OdigosEnvVarNamespace, ns)
 }
 
 func injectEnvVarObjectFieldRefToPodContainer(existingEnvNames *map[string]struct{}, container *corev1.Container, envVarName, envVarRef string) {