[mdatagen] Mdatagen link metrics and attributes from metadata to semconv

.chloggen/mdatagen-link-metrics-and-attributes-from-metadata-to-semconv.yaml

@@ -0,0 +1,28 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. receiver/otlp)
+component: cmd/mdatagen
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Add support for linking metrics to semantic conventions definitions
+
+# One or more tracking issues or pull requests related to the change
+issues: [13297]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext: |
+  Metrics in metadata.yaml can now reference semantic convention definitions via `semantic_convention.ref`.
+  The generated code will use semconv types for metric name, description, and unit.
+  Package and type are auto-inferred from the metric name (e.g., `system.cpu.time` → `systemconv.CPUTime`).
+
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: [user]

.github/workflows/utils/cspell.json

@@ -459,6 +459,7 @@
       "subpackages",
       "swiatekm",
       "syft",
+      "systemconv",
       "tailsampling",
       "tchannel",
       "telemetrygen",

cmd/mdatagen/go.mod

@@ -35,6 +35,7 @@ require (
 	go.uber.org/goleak v1.3.0
 	go.uber.org/zap v1.27.1
 	go.yaml.in/yaml/v3 v3.0.4
+	golang.org/x/mod v0.33.0
 	golang.org/x/text v0.34.0
 )
 

cmd/mdatagen/internal/loader.go

@@ -60,6 +60,8 @@ func LoadMetadata(filePath string) (Metadata, error) {
 		md.GeneratedPackageName = "metadata"
 	}
 
+	md.inferSemConvTypes()
+
 	if err := md.Validate(); err != nil {
 		return md, err
 	}

cmd/mdatagen/internal/loader_test.go

@@ -252,6 +252,23 @@ func TestLoadMetadata(t *testing.T) {
 						FullName:         "required_string_attr",
 						RequirementLevel: AttributeRequirementLevelRequired,
 					},
+					"cpu": {
+						Description: "Logical CPU number starting at 0.",
+						Type: ValueType{
+							ValueType: pcommon.ValueTypeStr,
+						},
+						FullName:         "cpu",
+						RequirementLevel: AttributeRequirementLevelRecommended,
+					},
+					"state": {
+						Description: "Breakdown of CPU usage by type.",
+						Enum:        []string{"idle", "interrupt", "nice", "softirq", "steal", "system", "user", "wait"},
+						Type: ValueType{
+							ValueType: pcommon.ValueTypeStr,
+						},
+						FullName:         "state",
+						RequirementLevel: AttributeRequirementLevelRecommended,
+					},
 				},
 				Metrics: map[MetricName]Metric{
 					"default.metric": {
@@ -302,9 +319,14 @@ func TestLoadMetadata(t *testing.T) {
 					},
 					"system.cpu.time": {
 						Signal: Signal{
-							Enabled:               true,
-							Stability:             component.StabilityLevelBeta,
-							SemanticConvention:    &SemanticConvention{SemanticConventionRef: "https://github.com/open-telemetry/semantic-conventions/blob/v1.38.0/docs/system/system-metrics.md#metric-systemcputime"},
+							Attributes: []AttributeName{"cpu", "state"},
+							Enabled:    true,
+							Stability:  component.StabilityLevelBeta,
+							SemanticConvention: &SemanticConvention{
+								SemanticConventionRef: "https://github.com/open-telemetry/semantic-conventions/blob/v1.38.0/docs/system/system-metrics.md#metric-systemcputime",
+								Package:               "systemconv",
+								Type:                  "CPUTime",
+							},
 							Description:           "Monotonic cumulative sum int metric enabled by default.",
 							ExtendedDocumentation: "The metric will be become optional soon.",
 						},
@@ -315,6 +337,24 @@ func TestLoadMetadata(t *testing.T) {
 							Mono:                   Mono{Monotonic: true},
 						},
 					},
+					"system.memory.limit": {
+						Signal: Signal{
+							Enabled:   true,
+							Stability: component.StabilityLevelDevelopment,
+							SemanticConvention: &SemanticConvention{
+								SemanticConventionRef: "https://github.com/open-telemetry/semantic-conventions/blob/v1.38.0/docs/system/system-metrics.md#metric-systemmemorylimit",
+								Package:               "systemconv",
+								Type:                  "MemoryLimit",
+							},
+							Description: "Total bytes of memory available.",
+						},
+						Unit: strPtr("By"),
+						Sum: &Sum{
+							MetricValueType:        MetricValueType{pmetric.NumberDataPointValueTypeInt},
+							AggregationTemporality: AggregationTemporality{Aggregation: pmetric.AggregationTemporalityCumulative},
+							Mono:                   Mono{Monotonic: false},
+						},
+					},
 					"optional.metric": {
 						Signal: Signal{
 							Enabled:     false,
@@ -608,6 +648,11 @@ func TestLoadMetadata(t *testing.T) {
 			want:    Metadata{},
 			wantErr: "metric \"default.metric\": invalid semantic-conventions URL: want https://github.com/open-telemetry/semantic-conventions/blob/v1.37.2/*#metric-defaultmetric, got \"https://github.com/open-telemetry/semantic-conventions/blob/v1.38.0/docs/system/system-metrics.md#metric-systemcputime\"",
 		},
+		{
+			name:    "testdata/invalid_metric_semconvref_old_version.yaml",
+			want:    Metadata{},
+			wantErr: "metric \"default.metric\": semantic_convention requires sem_conv_version >= 1.32.0",
+		},
 		{
 			name:    "testdata/no_metric_stability.yaml",
 			want:    Metadata{},
@@ -705,6 +750,8 @@ func TestLoadMetadata(t *testing.T) {
 							Stability:   component.StabilityLevelDevelopment,
 							SemanticConvention: &SemanticConvention{
 								SemanticConventionRef: "https://github.com/open-telemetry/semantic-conventions/blob/v1.38.0/docs/system/system-metrics.md#metric-systemdiskio_time",
+								Package:               "systemconv",
+								Type:                  "DiskIOTime",
 							},
 						},
 						Unit: strPtr("s"),

cmd/mdatagen/internal/metadata.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"regexp"
 	"slices"
+	"sort"
 	"strconv"
 	"strings"
 
@@ -527,6 +528,12 @@ func (mvt ValueType) Primitive() string {
 
 type SemanticConvention struct {
 	SemanticConventionRef string `mapstructure:"ref"`
+	// Package is the inferred semconv Go package name (e.g. "systemconv").
+	// This field is computed from the metric name, not user-specified.
+	Package string
+	// Type is the inferred semconv Go type name (e.g. "CPUTime").
+	// This field is computed from the metric name, not user-specified.
+	Type string
 }
 
 type Warnings struct {
@@ -702,6 +709,83 @@ func (s Signal) HasConditionalAttributes(attrs map[AttributeName]Attribute) bool
 	return false
 }
 
+func (s *SemanticConvention) HasSemConvType() bool {
+	return s != nil && s.Package != "" && s.Type != ""
+}
+
+func (s *SemanticConvention) ShouldUseSemConvValues() bool {
+	if s == nil || s.Type == "" {
+		return false
+	}
+	return true
+}
+
+func (s *SemanticConvention) ImportPath(semConvVersion string) string {
+	if s.Package == "" {
+		return ""
+	}
+	return fmt.Sprintf("go.opentelemetry.io/otel/semconv/%v/%s", semConvVersion, s.Package)
+}
+
+// inferSemConvTypes auto-populates Package and Type on any SemanticConvention
+// that has a ref URL, by inferring them from the metric name.
+func (md *Metadata) inferSemConvTypes() {
+	for name, m := range md.Metrics {
+		sc := m.SemanticConvention
+		if sc == nil || sc.SemanticConventionRef == "" {
+			continue
+		}
+		pkg, typeName, err := InferSemConvFromMetricName(string(name))
+		if err != nil {
+			// If inference fails, leave them empty; validation will catch it.
+			continue
+		}
+		sc.Package = pkg
+		sc.Type = typeName
+	}
+}
+
+// semconvAcronyms extends the golint acronyms with additional ones
+// used by the OTel semconv Go codegen that are not in the standard golint list.
+var semconvAcronyms = map[string]bool{
+	"IO": true,
+}
+
+// InferSemConvFromMetricName derives the semconv Go package and type name
+// from a dotted metric name. For example, "system.cpu.time" yields
+// package "systemconv" and type "CPUTime".
+func InferSemConvFromMetricName(metricName string) (pkg, typeName string, err error) {
+	parts := strings.Split(metricName, ".")
+	if len(parts) < 2 {
+		return "", "", fmt.Errorf("metric name %q must have at least 2 segments to infer semconv type", metricName)
+	}
+	pkg = parts[0] + "conv"
+
+	remaining := strings.Join(parts[1:], ".")
+	typeName, err = formatSemConvIdentifier(remaining)
+	return pkg, typeName, err
+}
+
+// formatSemConvIdentifier converts a dotted/underscored identifier into a
+// PascalCase Go identifier, applying both the standard golint acronyms and
+// the semconv-specific acronyms.
+func formatSemConvIdentifier(s string) (string, error) {
+	ident, err := FormatIdentifier(s, true)
+	if err != nil {
+		return "", err
+	}
+
+	// FormatIdentifier only knows about golint.Acronyms. We need to fix up
+	// any semconv-specific acronyms that it missed. We do this by scanning
+	// for Title-cased versions (e.g. "Io") and replacing them with the
+	// upper-case acronym (e.g. "IO") at word boundaries.
+	for acronym := range semconvAcronyms {
+		titleForm := strings.ToUpper(acronym[:1]) + strings.ToLower(acronym[1:])
+		ident = strings.ReplaceAll(ident, titleForm, acronym)
+	}
+	return ident, nil
+}
+
 type Entity struct {
 	// Type is the type of the entity.
 	Type string `mapstructure:"type"`
@@ -759,3 +843,31 @@ type FeatureGate struct {
 	// ReferenceURL is the URL with contextual information about the feature gate.
 	ReferenceURL string `mapstructure:"reference_url"`
 }
+
+type SemConvImport struct {
+	Package string
+	Alias   string
+}
+
+func (md Metadata) SemConvImports() []SemConvImport {
+	imports := make(map[string]SemConvImport)
+
+	for _, m := range md.Metrics {
+		if m.SemanticConvention != nil && m.SemanticConvention.Package != "" {
+			pkg := m.SemanticConvention.Package
+			imports[pkg] = SemConvImport{
+				Package: pkg,
+				Alias:   pkg,
+			}
+		}
+	}
+
+	result := make([]SemConvImport, 0, len(imports))
+	for _, imp := range imports {
+		result = append(result, imp)
+	}
+	sort.Slice(result, func(i, j int) bool {
+		return result[i].Package < result[j].Package
+	})
+	return result
+}