[receiver/elasticapmintake][processor/elasticapm] Update resource attribute mappings to use semconv to fix aggregated metrics

internal/elasticattr/attributes.go

@@ -42,13 +42,6 @@ const (
 	FaaSExecution          = "faas.execution"
 	FaaSTriggerRequestID   = "faas.trigger.request_id"
 	HostHostName           = "host.hostname"
-	HostOSPlatform         = "host.os.platform"
-	ProcessRuntimeName     = "process.runtime.name"
-	ProcessRuntimeVersion  = "process.runtime.version"
-	ServiceLanguageName    = "service.language.name"
-	ServiceLanguageVersion = "service.language.version"
-	ServiceRuntimeName     = "service.runtime.name"
-	ServiceRuntimeVersion  = "service.runtime.version"
 	ServiceOriginID        = "service.origin.id"
 	ServiceOriginName      = "service.origin.name"
 	ServiceOriginVersion   = "service.origin.version"

processor/elasticapmprocessor/internal/ecs/ecs_translation.go

@@ -21,10 +21,11 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/elastic/opentelemetry-collector-components/internal/elasticattr"
 	"go.opentelemetry.io/collector/pdata/pcommon"
 	semconv26 "go.opentelemetry.io/otel/semconv/v1.26.0"
 	semconv "go.opentelemetry.io/otel/semconv/v1.27.0"
+
+	"github.com/elastic/opentelemetry-collector-components/internal/elasticattr"
 )
 
 // Supported ECS resource attributes
@@ -181,12 +182,8 @@ func isSupportedAttribute(attr string) bool {
 		string(semconv.ServiceVersionKey),
 		string(semconv.ServiceInstanceIDKey),
 		string(semconv.ServiceNamespaceKey),
-		elasticattr.ServiceLanguageName,
-		elasticattr.ServiceLanguageVersion,
 		elasticattr.ServiceFrameworkName,
 		elasticattr.ServiceFrameworkVersion,
-		elasticattr.ServiceRuntimeName,
-		elasticattr.ServiceRuntimeVersion,
 		elasticattr.ServiceOriginID,
 		elasticattr.ServiceOriginName,
 		elasticattr.ServiceOriginVersion,
@@ -246,8 +243,7 @@ func isSupportedAttribute(attr string) bool {
 		string(semconv.HostIDKey),
 		string(semconv.HostTypeKey),
 		string(semconv.HostArchKey),
-		string(semconv.HostIPKey),
-		elasticattr.HostOSPlatform:
+		string(semconv.HostIPKey):
 		return true
 
 	// process.*
@@ -256,8 +252,8 @@ func isSupportedAttribute(attr string) bool {
 		string(semconv.ProcessExecutableNameKey),
 		string(semconv.ProcessCommandLineKey),
 		string(semconv.ProcessExecutablePathKey),
-		elasticattr.ProcessRuntimeName,
-		elasticattr.ProcessRuntimeVersion,
+		string(semconv.ProcessRuntimeNameKey),
+		string(semconv.ProcessRuntimeVersionKey),
 		string(semconv.ProcessOwnerKey):
 		return true
 

processor/elasticapmprocessor/internal/ecs/ecs_translation_test.go

@@ -117,6 +117,58 @@ func TestTranslateResourceMetadata(t *testing.T) {
 			wantKey:    "labels.x_y_z_w",
 			wantAbsent: `x.y*z"w`,
 		},
+		{
+			name:     "supported telemetry sdk language",
+			inputKey: string(semconv.TelemetrySDKLanguageKey),
+			inputVal: "dotnet",
+			wantKey:  string(semconv.TelemetrySDKLanguageKey),
+		},
+		{
+			name:     "supported telemetry sdk version",
+			inputKey: string(semconv.TelemetrySDKVersionKey),
+			inputVal: "8.0.0",
+			wantKey:  string(semconv.TelemetrySDKVersionKey),
+		},
+		{
+			name:     "supported process runtime name",
+			inputKey: string(semconv.ProcessRuntimeNameKey),
+			inputVal: "node",
+			wantKey:  string(semconv.ProcessRuntimeNameKey),
+		},
+		{
+			name:     "supported process runtime version",
+			inputKey: string(semconv.ProcessRuntimeVersionKey),
+			inputVal: "20.11.1",
+			wantKey:  string(semconv.ProcessRuntimeVersionKey),
+		},
+		{
+			name:       "legacy service language name moved to labels",
+			inputKey:   "service.language.name",
+			inputVal:   "ecmascript",
+			wantKey:    "labels.service_language_name",
+			wantAbsent: "service.language.name",
+		},
+		{
+			name:       "legacy service language version moved to labels",
+			inputKey:   "service.language.version",
+			inputVal:   "20",
+			wantKey:    "labels.service_language_version",
+			wantAbsent: "service.language.version",
+		},
+		{
+			name:       "legacy service runtime name moved to labels",
+			inputKey:   "service.runtime.name",
+			inputVal:   "node",
+			wantKey:    "labels.service_runtime_name",
+			wantAbsent: "service.runtime.name",
+		},
+		{
+			name:       "legacy service runtime version moved to labels",
+			inputKey:   "service.runtime.version",
+			inputVal:   "20.11.1",
+			wantKey:    "labels.service_runtime_version",
+			wantAbsent: "service.runtime.version",
+		},
 	}
 	for _, tc := range cases {
 		t.Run(tc.name, func(t *testing.T) {

processor/elasticapmprocessor/testdata/ecs/elastic_log/input.yaml

@@ -189,10 +189,6 @@ resourceLogs:
         - key: host.arch
           value:
             stringValue: amd64
-        # host.* elastic attributes
-        - key: host.os.platform
-          value:
-            stringValue: linux
         # process.* attributes
         - key: process.pid
           value:

processor/elasticapmprocessor/testdata/ecs/elastic_log/output.yaml

@@ -44,10 +44,10 @@ resourceLogs:
         - key: service.namespace
           value:
             stringValue: production
-        - key: service.language.name
+        - key: labels.service_language_name
           value:
             stringValue: go
-        - key: service.language.version
+        - key: labels.service_language_version
           value:
             stringValue: 1.21.0
         - key: service.framework.name
@@ -56,10 +56,10 @@ resourceLogs:
         - key: service.framework.version
           value:
             stringValue: 1.9.0
-        - key: service.runtime.name
+        - key: labels.service_runtime_name
           value:
             stringValue: go
-        - key: service.runtime.version
+        - key: labels.service_runtime_version
           value:
             stringValue: go1.21.0
         - key: service.origin.id
@@ -179,9 +179,6 @@ resourceLogs:
         - key: host.arch
           value:
             stringValue: amd64
-        - key: host.os.platform
-          value:
-            stringValue: linux
         - key: process.pid
           value:
             intValue: "12345"

processor/elasticapmprocessor/testdata/ecs/elastic_metric/input.yaml

@@ -186,10 +186,6 @@ resourceMetrics:
         - key: host.arch
           value:
             stringValue: amd64
-        # host.* elastic attributes
-        - key: host.os.platform
-          value:
-            stringValue: linux
         # process.* attributes
         - key: process.pid
           value:

processor/elasticapmprocessor/testdata/ecs/elastic_metric/output.yaml

@@ -145,9 +145,6 @@ resourceMetrics:
         - key: host.name
           value:
             stringValue: node-1
-        - key: host.os.platform
-          value:
-            stringValue: linux
         - key: host.type
           value:
             stringValue: t3.medium
@@ -179,6 +176,18 @@ resourceMetrics:
         - key: labels.other_value
           value:
             stringValue: foo
+        - key: labels.service_language_name
+          value:
+            stringValue: go
+        - key: labels.service_language_version
+          value:
+            stringValue: 1.21.0
+        - key: labels.service_runtime_name
+          value:
+            stringValue: go
+        - key: labels.service_runtime_version
+          value:
+            stringValue: go1.21.0
         - key: labels.unsupported_key
           value:
             stringValue: foo
@@ -257,12 +266,6 @@ resourceMetrics:
         - key: service.instance.id
           value:
             stringValue: instance-123
-        - key: service.language.name
-          value:
-            stringValue: go
-        - key: service.language.version
-          value:
-            stringValue: 1.21.0
         - key: service.name
           value:
             stringValue: test-service
@@ -278,12 +281,6 @@ resourceMetrics:
         - key: service.origin.version
           value:
             stringValue: 2.0.0
-        - key: service.runtime.name
-          value:
-            stringValue: go
-        - key: service.runtime.version
-          value:
-            stringValue: go1.21.0
         - key: service.target.name
           value:
             stringValue: target-service

processor/elasticapmprocessor/testdata/ecs/elastic_span_db/input.yaml

@@ -189,10 +189,6 @@ resourceSpans:
         - key: host.arch
           value:
             stringValue: amd64
-        # host.* elastic attributes
-        - key: host.os.platform
-          value:
-            stringValue: linux
         # process.* attributes
         - key: process.pid
           value:

processor/elasticapmprocessor/testdata/ecs/elastic_span_db/output.yaml

@@ -41,10 +41,10 @@ resourceSpans:
         - key: service.namespace
           value:
             stringValue: production
-        - key: service.language.name
+        - key: labels.service_language_name
           value:
             stringValue: go
-        - key: service.language.version
+        - key: labels.service_language_version
           value:
             stringValue: 1.21.0
         - key: service.framework.name
@@ -53,10 +53,10 @@ resourceSpans:
         - key: service.framework.version
           value:
             stringValue: 1.9.0
-        - key: service.runtime.name
+        - key: labels.service_runtime_name
           value:
             stringValue: go
-        - key: service.runtime.version
+        - key: labels.service_runtime_version
           value:
             stringValue: go1.21.0
         - key: service.origin.id
@@ -179,9 +179,6 @@ resourceSpans:
         - key: host.arch
           value:
             stringValue: amd64
-        - key: host.os.platform
-          value:
-            stringValue: linux
         - key: process.pid
           value:
             intValue: "12345"

receiver/elasticapmintakereceiver/internal/mappers/intakeV2ToDerivedFields.go

@@ -105,13 +105,6 @@ func SetDerivedResourceAttributes(event *modelpb.APMEvent, attributes pcommon.Ma
 		attributes.PutStr(elasticattr.AgentName, event.Agent.Name)
 		attributes.PutStr(elasticattr.AgentVersion, event.Agent.Version)
 	}
-
-	if event.Service != nil {
-		if event.Service.Language != nil {
-			putNonEmptyStr(attributes, elasticattr.ServiceLanguageName, event.Service.Language.Name)
-			putNonEmptyStr(attributes, elasticattr.ServiceLanguageVersion, event.Service.Language.Version)
-		}
-	}
 }
 
 // SetDerivedFieldsForMetrics sets fields that are NOT part of OTel for metrics. These fields are derived by the Enrichment lib in case of OTLP input

receiver/elasticapmintakereceiver/internal/mappers/intakeV2ToElasticSpecificFields.go

@@ -413,10 +413,6 @@ func SetElasticSpecificResourceAttributes(event *modelpb.APMEvent, attributesMap
 			putNonEmptyStr(attributesMap, elasticattr.ServiceFrameworkName, event.Service.Framework.Name)
 			putNonEmptyStr(attributesMap, elasticattr.ServiceFrameworkVersion, event.Service.Framework.Version)
 		}
-		if event.Service.Runtime != nil {
-			putNonEmptyStr(attributesMap, elasticattr.ServiceRuntimeName, event.Service.Runtime.Name)
-			putNonEmptyStr(attributesMap, elasticattr.ServiceRuntimeVersion, event.Service.Runtime.Version)
-		}
 		if event.Service.Origin != nil {
 			putNonEmptyStr(attributesMap, elasticattr.ServiceOriginID, event.Service.Origin.Id)
 			putNonEmptyStr(attributesMap, elasticattr.ServiceOriginName, event.Service.Origin.Name)
@@ -425,9 +421,6 @@ func SetElasticSpecificResourceAttributes(event *modelpb.APMEvent, attributesMap
 	}
 
 	if event.Host != nil {
-		if event.Host.Os != nil {
-			putNonEmptyStr(attributesMap, elasticattr.HostOSPlatform, event.Host.Os.Platform)
-		}
 		putNonEmptyStr(attributesMap, elasticattr.HostHostName, event.Host.Hostname)
 	}
 

receiver/elasticapmintakereceiver/internal/mappers/intakeV2ToSemConv.go

@@ -20,8 +20,6 @@
 package mappers // import "github.com/elastic/opentelemetry-collector-components/receiver/elasticapmintakereceiver/internal/mappers"
 
 import (
-	"strings"
-
 	"go.opentelemetry.io/collector/pdata/pcommon"
 	semconv22 "go.opentelemetry.io/otel/semconv/v1.22.0"
 	semconv "go.opentelemetry.io/otel/semconv/v1.27.0"
@@ -34,8 +32,13 @@ func TranslateToOtelResourceAttributes(event *modelpb.APMEvent, attributes pcomm
 	if event.Service != nil {
 		putNonEmptyStr(attributes, string(semconv.ServiceNameKey), event.Service.Name)
 		putNonEmptyStr(attributes, string(semconv.ServiceVersionKey), event.Service.Version)
-		if event.Service.Language != nil && event.Service.Language.Name != "" {
-			attributes.PutStr(string(semconv.TelemetrySDKLanguageKey), translateElasticServiceLanguageToOtelSdkLanguage(event.Service.Language.Name))
+		if event.Service.Language != nil {
+			putNonEmptyStr(attributes, string(semconv.TelemetrySDKLanguageKey), event.Service.Language.Name)
+			putNonEmptyStr(attributes, string(semconv.TelemetrySDKVersionKey), event.Service.Language.Version)
+		}
+		if event.Service.Runtime != nil {
+			putNonEmptyStr(attributes, string(semconv.ProcessRuntimeNameKey), event.Service.Runtime.Name)
+			putNonEmptyStr(attributes, string(semconv.ProcessRuntimeVersionKey), event.Service.Runtime.Version)
 		}
 		attributes.PutStr(string(semconv.TelemetrySDKNameKey), "ElasticAPM")
 		if event.Service.Environment != "" {
@@ -53,6 +56,7 @@ func TranslateToOtelResourceAttributes(event *modelpb.APMEvent, attributes pcomm
 		putNonEmptyStr(attributes, string(semconv.HostArchKey), event.Host.Architecture)
 		if event.Host.Os != nil {
 			putNonEmptyStr(attributes, string(semconv.OSNameKey), event.Host.Os.Name)
+			putNonEmptyStr(attributes, string(semconv.OSTypeKey), event.Host.Os.Platform)
 			putNonEmptyStr(attributes, string(semconv.OSVersionKey), event.Host.Os.Version)
 		}
 	}
@@ -70,19 +74,6 @@ func TranslateToOtelResourceAttributes(event *modelpb.APMEvent, attributes pcomm
 	translateFaasAttributes(event, attributes)
 }
 
-// SemConv defines a well known list of values of telemetry.sdk.language: https://opentelemetry.io/docs/specs/semconv/attributes-registry/telemetry/
-// The classic Elastic APM Agents report values that may not be in the SemConv well known list.
-// This method maps those values to the closest SemConv well known value.
-func translateElasticServiceLanguageToOtelSdkLanguage(language string) string {
-	language_lower_case := strings.ToLower(language)
-	switch language_lower_case {
-	case "c#":
-		return "dotnet"
-	default:
-		return language_lower_case
-	}
-}
-
 // TranslateIntakeV2TransactionToOTelAttributes translates transaction attributes from the Elastic APM model to SemConv attributes
 func TranslateIntakeV2TransactionToOTelAttributes(event *modelpb.APMEvent, attributes pcommon.Map) {
 	translateHttpAttributes(event, attributes)