diff --git a/.chloggen/mdatagen-link-metrics-and-attributes-from-metadata-to-semconv.yaml b/.chloggen/mdatagen-link-metrics-and-attributes-from-metadata-to-semconv.yaml new file mode 100644 index 000000000000..7bf730ef2a01 --- /dev/null +++ b/.chloggen/mdatagen-link-metrics-and-attributes-from-metadata-to-semconv.yaml @@ -0,0 +1,28 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. receiver/otlp) +component: cmd/mdatagen + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Add support for linking metrics to semantic conventions definitions + +# One or more tracking issues or pull requests related to the change +issues: [13297] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: | + Metrics in metadata.yaml can now reference semantic convention definitions via `semantic_convention.ref`. + The generated code will use semconv types for metric name, description, and unit. + Package and type are auto-inferred from the metric name (e.g., `system.cpu.time` → `systemconv.CPUTime`). + +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [user] diff --git a/.github/workflows/utils/cspell.json b/.github/workflows/utils/cspell.json index 8db53b22a41b..23fd5f7f1275 100644 --- a/.github/workflows/utils/cspell.json +++ b/.github/workflows/utils/cspell.json @@ -459,6 +459,7 @@ "subpackages", "swiatekm", "syft", + "systemconv", "tailsampling", "tchannel", "telemetrygen", diff --git a/cmd/mdatagen/go.mod b/cmd/mdatagen/go.mod index 854d03e1335f..78025b6ef3e7 100644 --- a/cmd/mdatagen/go.mod +++ b/cmd/mdatagen/go.mod @@ -35,6 +35,7 @@ require ( go.uber.org/goleak v1.3.0 go.uber.org/zap v1.27.1 go.yaml.in/yaml/v3 v3.0.4 + golang.org/x/mod v0.33.0 golang.org/x/text v0.34.0 ) diff --git a/cmd/mdatagen/go.sum b/cmd/mdatagen/go.sum index cb12cfb92ab4..7a125810d841 100644 --- a/cmd/mdatagen/go.sum +++ b/cmd/mdatagen/go.sum @@ -88,6 +88,8 @@ go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc= go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8= +golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w= golang.org/x/net v0.50.0 h1:ucWh9eiCGyDR3vtzso0WMQinm2Dnt8cFMuQa9K33J60= golang.org/x/net v0.50.0/go.mod h1:UgoSli3F/pBgdJBHCTc+tp3gmrU4XswgGRgtnwWTfyM= golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= diff --git a/cmd/mdatagen/internal/loader.go b/cmd/mdatagen/internal/loader.go index 159c1daada8e..35bc22a4eb3b 100644 --- a/cmd/mdatagen/internal/loader.go +++ b/cmd/mdatagen/internal/loader.go @@ -60,6 +60,8 @@ func LoadMetadata(filePath string) (Metadata, error) { md.GeneratedPackageName = "metadata" } + md.inferSemConvTypes() + if err := md.Validate(); err != nil { return md, err } diff --git a/cmd/mdatagen/internal/loader_test.go b/cmd/mdatagen/internal/loader_test.go index 5e9c4daa1aa4..09dd88354f6f 100644 --- a/cmd/mdatagen/internal/loader_test.go +++ b/cmd/mdatagen/internal/loader_test.go @@ -252,6 +252,23 @@ func TestLoadMetadata(t *testing.T) { FullName: "required_string_attr", RequirementLevel: AttributeRequirementLevelRequired, }, + "cpu": { + Description: "Logical CPU number starting at 0.", + Type: ValueType{ + ValueType: pcommon.ValueTypeStr, + }, + FullName: "cpu", + RequirementLevel: AttributeRequirementLevelRecommended, + }, + "state": { + Description: "Breakdown of CPU usage by type.", + Enum: []string{"idle", "interrupt", "nice", "softirq", "steal", "system", "user", "wait"}, + Type: ValueType{ + ValueType: pcommon.ValueTypeStr, + }, + FullName: "state", + RequirementLevel: AttributeRequirementLevelRecommended, + }, }, Metrics: map[MetricName]Metric{ "default.metric": { @@ -302,9 +319,14 @@ func TestLoadMetadata(t *testing.T) { }, "system.cpu.time": { Signal: Signal{ - Enabled: true, - Stability: component.StabilityLevelBeta, - SemanticConvention: &SemanticConvention{SemanticConventionRef: "https://github.com/open-telemetry/semantic-conventions/blob/v1.38.0/docs/system/system-metrics.md#metric-systemcputime"}, + Attributes: []AttributeName{"cpu", "state"}, + Enabled: true, + Stability: component.StabilityLevelBeta, + SemanticConvention: &SemanticConvention{ + SemanticConventionRef: "https://github.com/open-telemetry/semantic-conventions/blob/v1.38.0/docs/system/system-metrics.md#metric-systemcputime", + Package: "systemconv", + Type: "CPUTime", + }, Description: "Monotonic cumulative sum int metric enabled by default.", ExtendedDocumentation: "The metric will be become optional soon.", }, @@ -315,6 +337,24 @@ func TestLoadMetadata(t *testing.T) { Mono: Mono{Monotonic: true}, }, }, + "system.memory.limit": { + Signal: Signal{ + Enabled: true, + Stability: component.StabilityLevelDevelopment, + SemanticConvention: &SemanticConvention{ + SemanticConventionRef: "https://github.com/open-telemetry/semantic-conventions/blob/v1.38.0/docs/system/system-metrics.md#metric-systemmemorylimit", + Package: "systemconv", + Type: "MemoryLimit", + }, + Description: "Total bytes of memory available.", + }, + Unit: strPtr("By"), + Sum: &Sum{ + MetricValueType: MetricValueType{pmetric.NumberDataPointValueTypeInt}, + AggregationTemporality: AggregationTemporality{Aggregation: pmetric.AggregationTemporalityCumulative}, + Mono: Mono{Monotonic: false}, + }, + }, "optional.metric": { Signal: Signal{ Enabled: false, @@ -608,6 +648,11 @@ func TestLoadMetadata(t *testing.T) { want: Metadata{}, wantErr: "metric \"default.metric\": invalid semantic-conventions URL: want https://github.com/open-telemetry/semantic-conventions/blob/v1.37.2/*#metric-defaultmetric, got \"https://github.com/open-telemetry/semantic-conventions/blob/v1.38.0/docs/system/system-metrics.md#metric-systemcputime\"", }, + { + name: "testdata/invalid_metric_semconvref_old_version.yaml", + want: Metadata{}, + wantErr: "metric \"default.metric\": semantic_convention requires sem_conv_version >= 1.32.0", + }, { name: "testdata/no_metric_stability.yaml", want: Metadata{}, @@ -705,6 +750,8 @@ func TestLoadMetadata(t *testing.T) { Stability: component.StabilityLevelDevelopment, SemanticConvention: &SemanticConvention{ SemanticConventionRef: "https://github.com/open-telemetry/semantic-conventions/blob/v1.38.0/docs/system/system-metrics.md#metric-systemdiskio_time", + Package: "systemconv", + Type: "DiskIOTime", }, }, Unit: strPtr("s"), diff --git a/cmd/mdatagen/internal/metadata.go b/cmd/mdatagen/internal/metadata.go index 56ad688cb5ee..e59b673ca5f5 100644 --- a/cmd/mdatagen/internal/metadata.go +++ b/cmd/mdatagen/internal/metadata.go @@ -8,6 +8,7 @@ import ( "fmt" "regexp" "slices" + "sort" "strconv" "strings" @@ -527,6 +528,12 @@ func (mvt ValueType) Primitive() string { type SemanticConvention struct { SemanticConventionRef string `mapstructure:"ref"` + // Package is the inferred semconv Go package name (e.g. "systemconv"). + // This field is computed from the metric name, not user-specified. + Package string + // Type is the inferred semconv Go type name (e.g. "CPUTime"). + // This field is computed from the metric name, not user-specified. + Type string } type Warnings struct { @@ -702,6 +709,83 @@ func (s Signal) HasConditionalAttributes(attrs map[AttributeName]Attribute) bool return false } +func (s *SemanticConvention) HasSemConvType() bool { + return s != nil && s.Package != "" && s.Type != "" +} + +func (s *SemanticConvention) ShouldUseSemConvValues() bool { + if s == nil || s.Type == "" { + return false + } + return true +} + +func (s *SemanticConvention) ImportPath(semConvVersion string) string { + if s.Package == "" { + return "" + } + return fmt.Sprintf("go.opentelemetry.io/otel/semconv/%v/%s", semConvVersion, s.Package) +} + +// inferSemConvTypes auto-populates Package and Type on any SemanticConvention +// that has a ref URL, by inferring them from the metric name. +func (md *Metadata) inferSemConvTypes() { + for name, m := range md.Metrics { + sc := m.SemanticConvention + if sc == nil || sc.SemanticConventionRef == "" { + continue + } + pkg, typeName, err := InferSemConvFromMetricName(string(name)) + if err != nil { + // If inference fails, leave them empty; validation will catch it. + continue + } + sc.Package = pkg + sc.Type = typeName + } +} + +// semconvAcronyms extends the golint acronyms with additional ones +// used by the OTel semconv Go codegen that are not in the standard golint list. +var semconvAcronyms = map[string]bool{ + "IO": true, +} + +// InferSemConvFromMetricName derives the semconv Go package and type name +// from a dotted metric name. For example, "system.cpu.time" yields +// package "systemconv" and type "CPUTime". +func InferSemConvFromMetricName(metricName string) (pkg, typeName string, err error) { + parts := strings.Split(metricName, ".") + if len(parts) < 2 { + return "", "", fmt.Errorf("metric name %q must have at least 2 segments to infer semconv type", metricName) + } + pkg = parts[0] + "conv" + + remaining := strings.Join(parts[1:], ".") + typeName, err = formatSemConvIdentifier(remaining) + return pkg, typeName, err +} + +// formatSemConvIdentifier converts a dotted/underscored identifier into a +// PascalCase Go identifier, applying both the standard golint acronyms and +// the semconv-specific acronyms. +func formatSemConvIdentifier(s string) (string, error) { + ident, err := FormatIdentifier(s, true) + if err != nil { + return "", err + } + + // FormatIdentifier only knows about golint.Acronyms. We need to fix up + // any semconv-specific acronyms that it missed. We do this by scanning + // for Title-cased versions (e.g. "Io") and replacing them with the + // upper-case acronym (e.g. "IO") at word boundaries. + for acronym := range semconvAcronyms { + titleForm := strings.ToUpper(acronym[:1]) + strings.ToLower(acronym[1:]) + ident = strings.ReplaceAll(ident, titleForm, acronym) + } + return ident, nil +} + type Entity struct { // Type is the type of the entity. Type string `mapstructure:"type"` @@ -759,3 +843,31 @@ type FeatureGate struct { // ReferenceURL is the URL with contextual information about the feature gate. ReferenceURL string `mapstructure:"reference_url"` } + +type SemConvImport struct { + Package string + Alias string +} + +func (md Metadata) SemConvImports() []SemConvImport { + imports := make(map[string]SemConvImport) + + for _, m := range md.Metrics { + if m.SemanticConvention != nil && m.SemanticConvention.Package != "" { + pkg := m.SemanticConvention.Package + imports[pkg] = SemConvImport{ + Package: pkg, + Alias: pkg, + } + } + } + + result := make([]SemConvImport, 0, len(imports)) + for _, imp := range imports { + result = append(result, imp) + } + sort.Slice(result, func(i, j int) bool { + return result[i].Package < result[j].Package + }) + return result +} diff --git a/cmd/mdatagen/internal/metadata_test.go b/cmd/mdatagen/internal/metadata_test.go index 3e7fe2b885fc..af1843406574 100644 --- a/cmd/mdatagen/internal/metadata_test.go +++ b/cmd/mdatagen/internal/metadata_test.go @@ -558,3 +558,398 @@ func TestValidateFeatureGatesNotSorted(t *testing.T) { require.Error(t, err) assert.ErrorContains(t, err, "feature gates must be sorted by ID") } + +func TestInferSemConvFromMetricName(t *testing.T) { + tests := []struct { + name string + metric string + wantPkg string + wantType string + wantErr bool + }{ + { + name: "go.goroutine.count metric returns correct package and type", + metric: "go.goroutine.count", + wantPkg: "goconv", + wantType: "GoroutineCount", + }, + { + name: "k8s.container.cpu.limit metric returns correct package and type", + metric: "k8s.container.cpu.limit", + wantPkg: "k8sconv", + wantType: "ContainerCPULimit", + }, + { + name: "system.cpu.time metric returns correct package and type", + metric: "system.cpu.time", + wantPkg: "systemconv", + wantType: "CPUTime", + }, + { + name: "system.cpu.logical.count metric returns correct package and type", + metric: "system.cpu.logical.count", + wantPkg: "systemconv", + wantType: "CPULogicalCount", + }, + { + name: "system.memory.limit metric returns correct package and type", + metric: "system.memory.limit", + wantPkg: "systemconv", + wantType: "MemoryLimit", + }, + { + name: "system.disk.io uses IO acronym", + metric: "system.disk.io", + wantPkg: "systemconv", + wantType: "DiskIO", + }, + { + name: "system.disk.io_time handles underscores and IO", + metric: "system.disk.io_time", + wantPkg: "systemconv", + wantType: "DiskIOTime", + }, + { + name: "system.network.io metric returns correct package and type", + metric: "system.network.io", + wantPkg: "systemconv", + wantType: "NetworkIO", + }, + { + name: "system.linux.memory.available metric returns correct package and type", + metric: "system.linux.memory.available", + wantPkg: "systemconv", + wantType: "LinuxMemoryAvailable", + }, + { + name: "system.linux.memory.slab.usage metric returns correct package and type", + metric: "system.linux.memory.slab.usage", + wantPkg: "systemconv", + wantType: "LinuxMemorySlabUsage", + }, + { + name: "system.uptime single remaining segment", + metric: "system.uptime", + wantPkg: "systemconv", + wantType: "Uptime", + }, + { + name: "system.filesystem.utilization metric returns correct package and type", + metric: "system.filesystem.utilization", + wantPkg: "systemconv", + wantType: "FilesystemUtilization", + }, + { + name: "system.paging.faults metric returns correct package and type", + metric: "system.paging.faults", + wantPkg: "systemconv", + wantType: "PagingFaults", + }, + { + name: "system.network.packet.dropped metric returns correct package and type", + metric: "system.network.packet.dropped", + wantPkg: "systemconv", + wantType: "NetworkPacketDropped", + }, + { + name: "http.server.request.duration uses different namespace", + metric: "http.server.request.duration", + wantPkg: "httpconv", + wantType: "ServerRequestDuration", + }, + { + name: "single segment fails", + metric: "system", + wantErr: true, + }, + { + name: "empty string fails", + metric: "", + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + pkg, typeName, err := InferSemConvFromMetricName(tt.metric) + if tt.wantErr { + require.Error(t, err) + return + } + require.NoError(t, err) + assert.Equal(t, tt.wantPkg, pkg) + assert.Equal(t, tt.wantType, typeName) + }) + } +} + +func TestInferSemConvTypes(t *testing.T) { + tests := []struct { + name string + metricName string + md Metadata + wantPkg string + wantType string + wantNil bool + }{ + { + name: "system.cpu.time has valid package and type", + metricName: "system.cpu.time", + md: Metadata{ + Metrics: map[MetricName]Metric{ + "system.cpu.time": { + Signal: Signal{ + SemanticConvention: &SemanticConvention{ + SemanticConventionRef: "https://example.com", + }, + }, + }, + }, + }, + wantPkg: "systemconv", + wantType: "CPUTime", + }, + { + name: "system.disk.io has valid package and type", + metricName: "system.disk.io", + md: Metadata{ + Metrics: map[MetricName]Metric{ + "system.disk.io": { + Signal: Signal{ + SemanticConvention: &SemanticConvention{ + SemanticConventionRef: "https://example.com", + }, + }, + }, + }, + }, + wantPkg: "systemconv", + wantType: "DiskIO", + }, + { + name: "no semantic convention doesn't do anything", + metricName: "default.metric", + md: Metadata{ + Metrics: map[MetricName]Metric{ + "default.metric": { + Signal: Signal{}, + }, + }, + }, + wantNil: true, + }, + { + name: "invalid metric name doesn't pupulate pkg and type", + metricName: "invalid", + md: Metadata{ + Metrics: map[MetricName]Metric{ + "invalid": { + Signal: Signal{ + SemanticConvention: &SemanticConvention{ + SemanticConventionRef: "https://example.com", + }, + }, + }, + }, + }, + wantPkg: "", + wantType: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tt.md.inferSemConvTypes() + sc := tt.md.Metrics[MetricName(tt.metricName)].SemanticConvention + if tt.wantNil { + assert.Nil(t, sc) + require.Nil(t, sc, "The semantic convention must be nil") + } else { + require.NotNil(t, sc, "The semantic convention must not be nil") + assert.Equal(t, tt.wantPkg, sc.Package) + assert.Equal(t, tt.wantType, sc.Type) + } + }) + } +} + +func TestSemConvImports(t *testing.T) { + tests := []struct { + name string + md Metadata + want []SemConvImport + }{ + { + name: "empty semconv import", + md: Metadata{}, + want: []SemConvImport{}, + }, + { + name: "Multiple packages sorted alphabetically", + md: Metadata{ + Metrics: map[MetricName]Metric{ + "system.cpu.time": { + Signal: Signal{ + SemanticConvention: &SemanticConvention{ + Package: "systemconv", + }, + }, + }, + "http.server.duration": { + Signal: Signal{ + SemanticConvention: &SemanticConvention{ + Package: "httpconv", + }, + }, + }, + }, + }, + want: []SemConvImport{ + { + Package: "httpconv", + Alias: "httpconv", + }, + { + Package: "systemconv", + Alias: "systemconv", + }, + }, + }, + { + name: "duplicate packages deduplicates", + md: Metadata{ + Metrics: map[MetricName]Metric{ + "system.cpu.time": { + Signal: Signal{ + SemanticConvention: &SemanticConvention{ + Package: "systemconv", + }, + }, + }, + "system.memory.usage": { + Signal: Signal{ + SemanticConvention: &SemanticConvention{ + Package: "systemconv", + }, + }, + }, + }, + }, + want: []SemConvImport{ + { + Package: "systemconv", + Alias: "systemconv", + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tt.md.SemConvImports() + assert.Equal(t, tt.want, got) + }) + } +} + +func TestShouldUseSemConvValues(t *testing.T) { + tests := []struct { + name string + sc *SemanticConvention + want bool + }{ + { + name: "nil semantic convention", + sc: nil, + want: false, + }, + { + name: "empty type", + sc: &SemanticConvention{Type: ""}, + want: false, + }, + { + name: "type set returns true", + sc: &SemanticConvention{Type: "CPUTime"}, + want: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tt.sc.ShouldUseSemConvValues() + assert.Equal(t, tt.want, got) + }) + } +} + +func TestImportPath(t *testing.T) { + tests := []struct { + name string + sc *SemanticConvention + scv string + want string + }{ + { + name: "Empty package returns empty string", + sc: &SemanticConvention{ + Package: "", + }, + scv: "1.38.0", + want: "", + }, + { + name: "systemconv package", + sc: &SemanticConvention{ + Package: "systemconv", + }, + scv: "1.38.0", + want: "go.opentelemetry.io/otel/semconv/1.38.0/systemconv", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tt.sc.ImportPath(tt.scv) + assert.Equal(t, tt.want, got) + }) + } +} + +func TestFormatSemConvIdentifiers(t *testing.T) { + tests := []struct { + name string + semconvIdentifier string + want string + wantErr bool + }{ + { + name: "correctly formats semconv go type", + semconvIdentifier: "cpu.time", + want: "CPUTime", + }, + { + name: "correctly formats disk.io to DiskIO", + semconvIdentifier: "disk.io", + want: "DiskIO", + }, + { + name: "returns an error when an empty string is supplied", + semconvIdentifier: "", + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := formatSemConvIdentifier(tt.semconvIdentifier) + if tt.wantErr { + require.Error(t, err, "error is required for empty strings") + assert.Empty(t, got, "expect an empty string if an error is returned") + } else { + require.NoError(t, err, "require no error") + assert.Equal(t, tt.want, got) + } + }) + } +} diff --git a/cmd/mdatagen/internal/metric.go b/cmd/mdatagen/internal/metric.go index c1bcf9bf1a74..2cfa15f55ec5 100644 --- a/cmd/mdatagen/internal/metric.go +++ b/cmd/mdatagen/internal/metric.go @@ -9,6 +9,7 @@ import ( "regexp" "strings" + "golang.org/x/mod/semver" "golang.org/x/text/cases" "golang.org/x/text/language" @@ -89,6 +90,10 @@ func (m *Metric) validate(metricName MetricName, semConvVersion string) error { if m.Gauge != nil { errs = errors.Join(errs, m.Gauge.Validate()) } + if m.SemanticConvention != nil && semver.Compare("v"+semConvVersion, "v1.32.0") < 0 { + errs = errors.Join(errs, errors.New("semantic_convention requires sem_conv_version >= 1.32.0")) + return errs + } if m.SemanticConvention != nil { if err := validateSemConvMetricURL(m.SemanticConvention.SemanticConventionRef, semConvVersion, string(metricName)); err != nil { errs = errors.Join(errs, err) diff --git a/cmd/mdatagen/internal/sampleconnector/internal/metadata/generated_metrics_test.go b/cmd/mdatagen/internal/sampleconnector/internal/metadata/generated_metrics_test.go index bea4933d9162..b0739dc9c5f8 100644 --- a/cmd/mdatagen/internal/sampleconnector/internal/metadata/generated_metrics_test.go +++ b/cmd/mdatagen/internal/sampleconnector/internal/metadata/generated_metrics_test.go @@ -502,3 +502,13 @@ func TestMetricsBuilder(t *testing.T) { }) } } + +func TestAttributeEnumAttrStringInvalid(t *testing.T) { + assert.Equal(t, "", AttributeEnumAttr(999).String()) +} + +func TestMapAttributeEnumAttr(t *testing.T) { + for str, val := range MapAttributeEnumAttr { + assert.Equal(t, str, val.String()) + } +} diff --git a/cmd/mdatagen/internal/samplereceiver/documentation.md b/cmd/mdatagen/internal/samplereceiver/documentation.md index 363d558a0b16..13b9d86b0cda 100644 --- a/cmd/mdatagen/internal/samplereceiver/documentation.md +++ b/cmd/mdatagen/internal/samplereceiver/documentation.md @@ -108,6 +108,21 @@ The metric will be become optional soon. | ---- | ----------- | ---------- | ----------------------- | --------- | --------- | ------------------- | | s | Sum | Int | Cumulative | true | Beta | [system.cpu.time](https://github.com/open-telemetry/semantic-conventions/blob/v1.38.0/docs/system/system-metrics.md#metric-systemcputime) | +#### Attributes + +| Name | Description | Values | Requirement Level | +| ---- | ----------- | ------ | -------- | +| cpu | Logical CPU number starting at 0. | Any Str | Recommended | +| state | Breakdown of CPU usage by type. | Str: ``idle``, ``interrupt``, ``nice``, ``softirq``, ``steal``, ``system``, ``user``, ``wait`` | Recommended | + +### system.memory.limit + +Total bytes of memory available. + +| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | Stability | Semantic Convention | +| ---- | ----------- | ---------- | ----------------------- | --------- | --------- | ------------------- | +| By | Sum | Int | Cumulative | false | Development | [system.memory.limit](https://github.com/open-telemetry/semantic-conventions/blob/v1.38.0/docs/system/system-metrics.md#metric-systemmemorylimit) | + ## Optional Metrics The following metrics are not emitted by default. Each of them can be enabled by applying the following configuration: diff --git a/cmd/mdatagen/internal/samplereceiver/internal/metadata/generated_config.go b/cmd/mdatagen/internal/samplereceiver/internal/metadata/generated_config.go index 394caca2b59c..2eff0f516193 100644 --- a/cmd/mdatagen/internal/samplereceiver/internal/metadata/generated_config.go +++ b/cmd/mdatagen/internal/samplereceiver/internal/metadata/generated_config.go @@ -68,6 +68,7 @@ type MetricsConfig struct { ReaggregateMetric MetricConfig `mapstructure:"reaggregate.metric"` ReaggregateMetricWithRequired MetricConfig `mapstructure:"reaggregate.metric.with_required"` SystemCPUTime MetricConfig `mapstructure:"system.cpu.time"` + SystemMemoryLimit MetricConfig `mapstructure:"system.memory.limit"` } func DefaultMetricsConfig() MetricsConfig { @@ -131,6 +132,14 @@ func DefaultMetricsConfig() MetricsConfig { SystemCPUTime: MetricConfig{ Enabled: true, + AggregationStrategy: AggregationStrategySum, + requiredAttributes: []string{}, + definedAttributes: []string{"cpu", "state"}, + EnabledAttributes: []string{"cpu", "state"}, + }, + SystemMemoryLimit: MetricConfig{ + Enabled: true, + AggregationStrategy: AggregationStrategySum, requiredAttributes: []string{}, definedAttributes: []string{}, diff --git a/cmd/mdatagen/internal/samplereceiver/internal/metadata/generated_config_test.go b/cmd/mdatagen/internal/samplereceiver/internal/metadata/generated_config_test.go index dd90f1d08603..1c45227c12c7 100644 --- a/cmd/mdatagen/internal/samplereceiver/internal/metadata/generated_config_test.go +++ b/cmd/mdatagen/internal/samplereceiver/internal/metadata/generated_config_test.go @@ -63,6 +63,11 @@ func TestMetricsBuilderConfig(t *testing.T) { EnabledAttributes: []string{"required_string_attr", "string_attr", "boolean_attr"}, }, SystemCPUTime: MetricConfig{ + Enabled: true, + AggregationStrategy: AggregationStrategySum, + EnabledAttributes: []string{"cpu", "state"}, + }, + SystemMemoryLimit: MetricConfig{ Enabled: true, AggregationStrategy: AggregationStrategySum, EnabledAttributes: []string{}, @@ -120,6 +125,11 @@ func TestMetricsBuilderConfig(t *testing.T) { EnabledAttributes: []string{"required_string_attr", "string_attr", "boolean_attr"}, }, SystemCPUTime: MetricConfig{ + Enabled: false, + AggregationStrategy: AggregationStrategySum, + EnabledAttributes: []string{"cpu", "state"}, + }, + SystemMemoryLimit: MetricConfig{ Enabled: false, AggregationStrategy: AggregationStrategySum, EnabledAttributes: []string{}, diff --git a/cmd/mdatagen/internal/samplereceiver/internal/metadata/generated_metrics.go b/cmd/mdatagen/internal/samplereceiver/internal/metadata/generated_metrics.go index aa7f1cea5032..c7a231460b4f 100644 --- a/cmd/mdatagen/internal/samplereceiver/internal/metadata/generated_metrics.go +++ b/cmd/mdatagen/internal/samplereceiver/internal/metadata/generated_metrics.go @@ -9,6 +9,7 @@ import ( "time" conventions "go.opentelemetry.io/otel/semconv/v1.38.0" + systemconv "go.opentelemetry.io/otel/semconv/v1.38.0/systemconv" "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/filter" @@ -54,6 +55,56 @@ var MapAttributeEnumAttr = map[string]AttributeEnumAttr{ "blue": AttributeEnumAttrBlue, } +// AttributeState specifies the value state attribute. +type AttributeState int + +const ( + _ AttributeState = iota + AttributeStateIdle + AttributeStateInterrupt + AttributeStateNice + AttributeStateSoftirq + AttributeStateSteal + AttributeStateSystem + AttributeStateUser + AttributeStateWait +) + +// String returns the string representation of the AttributeState. +func (av AttributeState) String() string { + switch av { + case AttributeStateIdle: + return "idle" + case AttributeStateInterrupt: + return "interrupt" + case AttributeStateNice: + return "nice" + case AttributeStateSoftirq: + return "softirq" + case AttributeStateSteal: + return "steal" + case AttributeStateSystem: + return "system" + case AttributeStateUser: + return "user" + case AttributeStateWait: + return "wait" + } + return "" +} + +// MapAttributeState is a helper map of string to AttributeState attribute value. +var MapAttributeState = map[string]AttributeState{ + "idle": AttributeStateIdle, + "interrupt": AttributeStateInterrupt, + "nice": AttributeStateNice, + "softirq": AttributeStateSoftirq, + "steal": AttributeStateSteal, + "system": AttributeStateSystem, + "user": AttributeStateUser, + "wait": AttributeStateWait, +} + var MetricsInfo = metricsInfo{ DefaultMetric: metricInfo{ Name: "default.metric", @@ -77,7 +128,10 @@ var MetricsInfo = metricsInfo{ Name: "reaggregate.metric.with_required", }, SystemCPUTime: metricInfo{ - Name: "system.cpu.time", + Name: metricSemConvSystemCPUTime.Name(), + }, + SystemMemoryLimit: metricInfo{ + Name: metricSemConvSystemMemoryLimit.Name(), }, } @@ -90,6 +144,7 @@ type metricsInfo struct { ReaggregateMetric metricInfo ReaggregateMetricWithRequired metricInfo SystemCPUTime metricInfo + SystemMemoryLimit metricInfo } type metricInfo struct { @@ -801,18 +856,22 @@ type metricSystemCPUTime struct { aggDataPoints []int64 // slice containing number of aggregated datapoints at each index } +// metricSemConvSystemCPUTime provides access to the semantic convention type +var metricSemConvSystemCPUTime = systemconv.CPUTime{} + // init fills system.cpu.time metric with initial data. func (m *metricSystemCPUTime) init() { - m.data.SetName("system.cpu.time") - m.data.SetDescription("Monotonic cumulative sum int metric enabled by default.") - m.data.SetUnit("s") + m.data.SetName(metricSemConvSystemCPUTime.Name()) + m.data.SetDescription(metricSemConvSystemCPUTime.Description()) + m.data.SetUnit(metricSemConvSystemCPUTime.Unit()) m.data.SetEmptySum() m.data.Sum().SetIsMonotonic(true) m.data.Sum().SetAggregationTemporality(pmetric.AggregationTemporalityCumulative) + m.data.Sum().DataPoints().EnsureCapacity(m.capacity) m.aggDataPoints = m.aggDataPoints[:0] } -func (m *metricSystemCPUTime) recordDataPoint(start pcommon.Timestamp, ts pcommon.Timestamp, val int64) { +func (m *metricSystemCPUTime) recordDataPoint(start pcommon.Timestamp, ts pcommon.Timestamp, val int64, cpuAttributeValue string, stateAttributeValue string) { if !m.config.Enabled { return } @@ -820,6 +879,12 @@ func (m *metricSystemCPUTime) recordDataPoint(start pcommon.Timestamp, ts pcommo dp := pmetric.NewNumberDataPoint() dp.SetStartTimestamp(start) dp.SetTimestamp(ts) + if slices.Contains(m.config.EnabledAttributes, "cpu") { + dp.Attributes().PutStr("cpu", cpuAttributeValue) + } + if slices.Contains(m.config.EnabledAttributes, "state") { + dp.Attributes().PutStr("state", stateAttributeValue) + } var s string dps := m.data.Sum().DataPoints() @@ -881,6 +946,96 @@ func newMetricSystemCPUTime(cfg MetricConfig) metricSystemCPUTime { return m } +type metricSystemMemoryLimit struct { + data pmetric.Metric // data buffer for generated metric. + config MetricConfig // metric config provided by user. + capacity int // max observed number of data points added to the metric. + aggDataPoints []int64 // slice containing number of aggregated datapoints at each index +} + +// metricSemConvSystemMemoryLimit provides access to the semantic convention type +var metricSemConvSystemMemoryLimit = systemconv.MemoryLimit{} + +// init fills system.memory.limit metric with initial data. +func (m *metricSystemMemoryLimit) init() { + m.data.SetName(metricSemConvSystemMemoryLimit.Name()) + m.data.SetDescription(metricSemConvSystemMemoryLimit.Description()) + m.data.SetUnit(metricSemConvSystemMemoryLimit.Unit()) + m.data.SetEmptySum() + m.data.Sum().SetIsMonotonic(false) + m.data.Sum().SetAggregationTemporality(pmetric.AggregationTemporalityCumulative) + m.aggDataPoints = m.aggDataPoints[:0] +} + +func (m *metricSystemMemoryLimit) recordDataPoint(start pcommon.Timestamp, ts pcommon.Timestamp, val int64) { + if !m.config.Enabled { + return + } + + dp := pmetric.NewNumberDataPoint() + dp.SetStartTimestamp(start) + dp.SetTimestamp(ts) + + var s string + dps := m.data.Sum().DataPoints() + for i := 0; i < dps.Len(); i++ { + dpi := dps.At(i) + if dp.Attributes().Equal(dpi.Attributes()) && dp.StartTimestamp() == dpi.StartTimestamp() && dp.Timestamp() == dpi.Timestamp() { + switch s = m.config.AggregationStrategy; s { + case AggregationStrategySum, AggregationStrategyAvg: + dpi.SetIntValue(dpi.IntValue() + val) + m.aggDataPoints[i] += 1 + return + case AggregationStrategyMin: + if dpi.IntValue() > val { + dpi.SetIntValue(val) + } + return + case AggregationStrategyMax: + if dpi.IntValue() < val { + dpi.SetIntValue(val) + } + return + } + } + } + + dp.SetIntValue(val) + m.aggDataPoints = append(m.aggDataPoints, 1) + dp.MoveTo(dps.AppendEmpty()) +} + +// updateCapacity saves max length of data point slices that will be used for the slice capacity. +func (m *metricSystemMemoryLimit) updateCapacity() { + if m.data.Sum().DataPoints().Len() > m.capacity { + m.capacity = m.data.Sum().DataPoints().Len() + } +} + +// emit appends recorded metric data to a metrics slice and prepares it for recording another set of data points. +func (m *metricSystemMemoryLimit) emit(metrics pmetric.MetricSlice) { + if m.config.Enabled && m.data.Sum().DataPoints().Len() > 0 { + if m.config.AggregationStrategy == AggregationStrategyAvg { + for i, aggCount := range m.aggDataPoints { + m.data.Sum().DataPoints().At(i).SetIntValue(m.data.Sum().DataPoints().At(i).IntValue() / aggCount) + } + } + m.updateCapacity() + m.data.MoveTo(metrics.AppendEmpty()) + m.init() + } +} + +func newMetricSystemMemoryLimit(cfg MetricConfig) metricSystemMemoryLimit { + m := metricSystemMemoryLimit{config: cfg} + + if cfg.Enabled { + m.data = pmetric.NewMetric() + m.init() + } + return m +} + // MetricsBuilder provides an interface for scrapers to report metrics while taking care of all the transformations // required to produce metric representation defined in metadata and user config. type MetricsBuilder struct { @@ -899,6 +1054,7 @@ type MetricsBuilder struct { metricReaggregateMetric metricReaggregateMetric metricReaggregateMetricWithRequired metricReaggregateMetricWithRequired metricSystemCPUTime metricSystemCPUTime + metricSystemMemoryLimit metricSystemMemoryLimit } // MetricBuilderOption applies changes to default metrics builder. @@ -953,6 +1109,7 @@ func NewMetricsBuilder(mbc MetricsBuilderConfig, settings receiver.Settings, opt metricReaggregateMetric: newMetricReaggregateMetric(mbc.Metrics.ReaggregateMetric), metricReaggregateMetricWithRequired: newMetricReaggregateMetricWithRequired(mbc.Metrics.ReaggregateMetricWithRequired), metricSystemCPUTime: newMetricSystemCPUTime(mbc.Metrics.SystemCPUTime), + metricSystemMemoryLimit: newMetricSystemMemoryLimit(mbc.Metrics.SystemMemoryLimit), resourceAttributeIncludeFilter: make(map[string]filter.Filter), resourceAttributeExcludeFilter: make(map[string]filter.Filter), } @@ -1082,6 +1239,7 @@ func (mb *MetricsBuilder) EmitForResource(options ...ResourceMetricsOption) { mb.metricReaggregateMetric.emit(ils.Metrics()) mb.metricReaggregateMetricWithRequired.emit(ils.Metrics()) mb.metricSystemCPUTime.emit(ils.Metrics()) + mb.metricSystemMemoryLimit.emit(ils.Metrics()) for _, op := range options { op.apply(rm) @@ -1154,8 +1312,13 @@ func (mb *MetricsBuilder) RecordReaggregateMetricWithRequiredDataPoint(ts pcommo } // RecordSystemCPUTimeDataPoint adds a data point to system.cpu.time metric. -func (mb *MetricsBuilder) RecordSystemCPUTimeDataPoint(ts pcommon.Timestamp, val int64) { - mb.metricSystemCPUTime.recordDataPoint(mb.startTime, ts, val) +func (mb *MetricsBuilder) RecordSystemCPUTimeDataPoint(ts pcommon.Timestamp, val int64, cpuAttributeValue string, stateAttributeValue AttributeState) { + mb.metricSystemCPUTime.recordDataPoint(mb.startTime, ts, val, cpuAttributeValue, stateAttributeValue.String()) +} + +// RecordSystemMemoryLimitDataPoint adds a data point to system.memory.limit metric. +func (mb *MetricsBuilder) RecordSystemMemoryLimitDataPoint(ts pcommon.Timestamp, val int64) { + mb.metricSystemMemoryLimit.recordDataPoint(mb.startTime, ts, val) } // Reset resets metrics builder to its initial state. It should be used when external metrics source is restarted, diff --git a/cmd/mdatagen/internal/samplereceiver/internal/metadata/generated_metrics_test.go b/cmd/mdatagen/internal/samplereceiver/internal/metadata/generated_metrics_test.go index db4e7257c688..0ef1640955c6 100644 --- a/cmd/mdatagen/internal/samplereceiver/internal/metadata/generated_metrics_test.go +++ b/cmd/mdatagen/internal/samplereceiver/internal/metadata/generated_metrics_test.go @@ -76,6 +76,7 @@ func TestMetricsBuilder(t *testing.T) { aggMap["ReaggregateMetric"] = mb.metricReaggregateMetric.config.AggregationStrategy aggMap["ReaggregateMetricWithRequired"] = mb.metricReaggregateMetricWithRequired.config.AggregationStrategy aggMap["SystemCPUTime"] = mb.metricSystemCPUTime.config.AggregationStrategy + aggMap["SystemMemoryLimit"] = mb.metricSystemMemoryLimit.config.AggregationStrategy expectedWarnings := 0 if tt.metricsSet == testDataSetDefault { @@ -162,9 +163,16 @@ func TestMetricsBuilder(t *testing.T) { defaultMetricsCount++ allMetricsCount++ - mb.RecordSystemCPUTimeDataPoint(ts, 1) + mb.RecordSystemCPUTimeDataPoint(ts, 1, "cpu-val", AttributeStateIdle) if tt.name == "reaggregate_set" { - mb.RecordSystemCPUTimeDataPoint(ts, 3) + mb.RecordSystemCPUTimeDataPoint(ts, 3, "cpu-val-2", AttributeStateInterrupt) + } + + defaultMetricsCount++ + allMetricsCount++ + mb.RecordSystemMemoryLimitDataPoint(ts, 1) + if tt.name == "reaggregate_set" { + mb.RecordSystemMemoryLimitDataPoint(ts, 3) } rb := mb.NewResourceBuilder() @@ -187,6 +195,7 @@ func TestMetricsBuilder(t *testing.T) { assert.Empty(t, mb.metricReaggregateMetric.aggDataPoints) assert.Empty(t, mb.metricReaggregateMetricWithRequired.aggDataPoints) assert.Empty(t, mb.metricSystemCPUTime.aggDataPoints) + assert.Empty(t, mb.metricSystemMemoryLimit.aggDataPoints) } if tt.expectEmpty { @@ -582,7 +591,7 @@ func TestMetricsBuilder(t *testing.T) { validatedMetrics["system.cpu.time"] = true assert.Equal(t, pmetric.MetricTypeSum, ms.At(i).Type()) assert.Equal(t, 1, ms.At(i).Sum().DataPoints().Len()) - assert.Equal(t, "Monotonic cumulative sum int metric enabled by default.", ms.At(i).Description()) + assert.Equal(t, metricSemConvSystemCPUTime.Description(), ms.At(i).Description()) assert.Equal(t, "s", ms.At(i).Unit()) assert.True(t, ms.At(i).Sum().IsMonotonic()) assert.Equal(t, pmetric.AggregationTemporalityCumulative, ms.At(i).Sum().AggregationTemporality()) @@ -591,12 +600,18 @@ func TestMetricsBuilder(t *testing.T) { assert.Equal(t, ts, dp.Timestamp()) assert.Equal(t, pmetric.NumberDataPointValueTypeInt, dp.ValueType()) assert.Equal(t, int64(1), dp.IntValue()) + attrVal, ok := dp.Attributes().Get("cpu") + assert.True(t, ok) + assert.Equal(t, "cpu-val", attrVal.Str()) + attrVal, ok = dp.Attributes().Get("state") + assert.True(t, ok) + assert.Equal(t, "idle", attrVal.Str()) } else { assert.False(t, validatedMetrics["system.cpu.time"], "Found a duplicate in the metrics slice: system.cpu.time") validatedMetrics["system.cpu.time"] = true assert.Equal(t, pmetric.MetricTypeSum, ms.At(i).Type()) assert.Equal(t, 1, ms.At(i).Sum().DataPoints().Len()) - assert.Equal(t, "Monotonic cumulative sum int metric enabled by default.", ms.At(i).Description()) + assert.Equal(t, metricSemConvSystemCPUTime.Description(), ms.At(i).Description()) assert.Equal(t, "s", ms.At(i).Unit()) assert.True(t, ms.At(i).Sum().IsMonotonic()) assert.Equal(t, pmetric.AggregationTemporalityCumulative, ms.At(i).Sum().AggregationTemporality()) @@ -614,9 +629,72 @@ func TestMetricsBuilder(t *testing.T) { case "max": assert.Equal(t, int64(3), dp.IntValue()) } + _, ok := dp.Attributes().Get("cpu") + assert.False(t, ok) + _, ok = dp.Attributes().Get("state") + assert.False(t, ok) + } + case "system.memory.limit": + if tt.name != "reaggregate_set" { + assert.False(t, validatedMetrics["system.memory.limit"], "Found a duplicate in the metrics slice: system.memory.limit") + validatedMetrics["system.memory.limit"] = true + assert.Equal(t, pmetric.MetricTypeSum, ms.At(i).Type()) + assert.Equal(t, 1, ms.At(i).Sum().DataPoints().Len()) + assert.Equal(t, metricSemConvSystemMemoryLimit.Description(), ms.At(i).Description()) + assert.Equal(t, "By", ms.At(i).Unit()) + assert.False(t, ms.At(i).Sum().IsMonotonic()) + assert.Equal(t, pmetric.AggregationTemporalityCumulative, ms.At(i).Sum().AggregationTemporality()) + dp := ms.At(i).Sum().DataPoints().At(0) + assert.Equal(t, start, dp.StartTimestamp()) + assert.Equal(t, ts, dp.Timestamp()) + assert.Equal(t, pmetric.NumberDataPointValueTypeInt, dp.ValueType()) + assert.Equal(t, int64(1), dp.IntValue()) + } else { + assert.False(t, validatedMetrics["system.memory.limit"], "Found a duplicate in the metrics slice: system.memory.limit") + validatedMetrics["system.memory.limit"] = true + assert.Equal(t, pmetric.MetricTypeSum, ms.At(i).Type()) + assert.Equal(t, 1, ms.At(i).Sum().DataPoints().Len()) + assert.Equal(t, metricSemConvSystemMemoryLimit.Description(), ms.At(i).Description()) + assert.Equal(t, "By", ms.At(i).Unit()) + assert.False(t, ms.At(i).Sum().IsMonotonic()) + assert.Equal(t, pmetric.AggregationTemporalityCumulative, ms.At(i).Sum().AggregationTemporality()) + dp := ms.At(i).Sum().DataPoints().At(0) + assert.Equal(t, start, dp.StartTimestamp()) + assert.Equal(t, ts, dp.Timestamp()) + assert.Equal(t, pmetric.NumberDataPointValueTypeInt, dp.ValueType()) + switch aggMap["system.memory.limit"] { + case "sum": + assert.Equal(t, int64(4), dp.IntValue()) + case "avg": + assert.Equal(t, int64(2), dp.IntValue()) + case "min": + assert.Equal(t, int64(1), dp.IntValue()) + case "max": + assert.Equal(t, int64(3), dp.IntValue()) + } } } } }) } } + +func TestAttributeEnumAttrStringInvalid(t *testing.T) { + assert.Equal(t, "", AttributeEnumAttr(999).String()) +} + +func TestMapAttributeEnumAttr(t *testing.T) { + for str, val := range MapAttributeEnumAttr { + assert.Equal(t, str, val.String()) + } +} + +func TestAttributeStateStringInvalid(t *testing.T) { + assert.Equal(t, "", AttributeState(999).String()) +} + +func TestMapAttributeState(t *testing.T) { + for str, val := range MapAttributeState { + assert.Equal(t, str, val.String()) + } +} diff --git a/cmd/mdatagen/internal/samplereceiver/internal/metadata/testdata/config.yaml b/cmd/mdatagen/internal/samplereceiver/internal/metadata/testdata/config.yaml index 9d11a3a39cd8..120cdd2e2427 100644 --- a/cmd/mdatagen/internal/samplereceiver/internal/metadata/testdata/config.yaml +++ b/cmd/mdatagen/internal/samplereceiver/internal/metadata/testdata/config.yaml @@ -23,6 +23,9 @@ all_set: enabled: true attributes: ["required_string_attr","string_attr","boolean_attr"] system.cpu.time: + enabled: true + attributes: ["cpu","state"] + system.memory.limit: enabled: true attributes: [] events: @@ -75,6 +78,9 @@ reaggregate_set: system.cpu.time: enabled: true attributes: [] + system.memory.limit: + enabled: true + attributes: [] events: default.event: enabled: true @@ -123,6 +129,9 @@ none_set: enabled: false attributes: ["required_string_attr","string_attr","boolean_attr"] system.cpu.time: + enabled: false + attributes: ["cpu","state"] + system.memory.limit: enabled: false attributes: [] events: diff --git a/cmd/mdatagen/internal/samplereceiver/metadata.yaml b/cmd/mdatagen/internal/samplereceiver/metadata.yaml index 2c057696b40d..5d107105a85c 100644 --- a/cmd/mdatagen/internal/samplereceiver/metadata.yaml +++ b/cmd/mdatagen/internal/samplereceiver/metadata.yaml @@ -104,6 +104,10 @@ attributes: type: string requirement_level: conditionally_required + cpu: + description: Logical CPU number starting at 0. + type: string + enum_attr: description: Attribute with a known set of string values. type: string @@ -132,6 +136,11 @@ attributes: description: Attribute with a slice value. type: slice + state: + description: Breakdown of CPU usage by type. + type: string + enum: [idle, interrupt, nice, softirq, steal, system, user, wait] + string_attr: description: Attribute with any string value. type: string @@ -284,9 +293,23 @@ metrics: value_type: int monotonic: true aggregation_temporality: cumulative + attributes: [cpu, state] semantic_convention: ref: https://github.com/open-telemetry/semantic-conventions/blob/v1.38.0/docs/system/system-metrics.md#metric-systemcputime + system.memory.limit: + enabled: true + stability: development + description: Total bytes of memory available. + unit: By + sum: + value_type: int + monotonic: false + aggregation_temporality: cumulative + # package and type are auto-inferred from the metric name + semantic_convention: + ref: https://github.com/open-telemetry/semantic-conventions/blob/v1.38.0/docs/system/system-metrics.md#metric-systemmemorylimit + telemetry: metrics: batch_size_trigger_send: diff --git a/cmd/mdatagen/internal/samplescraper/internal/metadata/generated_metrics.go b/cmd/mdatagen/internal/samplescraper/internal/metadata/generated_metrics.go index 9e703cc01ffe..692165ae5729 100644 --- a/cmd/mdatagen/internal/samplescraper/internal/metadata/generated_metrics.go +++ b/cmd/mdatagen/internal/samplescraper/internal/metadata/generated_metrics.go @@ -9,6 +9,7 @@ import ( "time" conventions "go.opentelemetry.io/otel/semconv/v1.38.0" + systemconv "go.opentelemetry.io/otel/semconv/v1.38.0/systemconv" "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/filter" @@ -74,7 +75,7 @@ var MetricsInfo = metricsInfo{ Name: "reaggregate.metric", }, SystemCPUTime: metricInfo{ - Name: "system.cpu.time", + Name: metricSemConvSystemCPUTime.Name(), }, } @@ -671,11 +672,14 @@ type metricSystemCPUTime struct { aggDataPoints []int64 // slice containing number of aggregated datapoints at each index } +// metricSemConvSystemCPUTime provides access to the semantic convention type +var metricSemConvSystemCPUTime = systemconv.CPUTime{} + // init fills system.cpu.time metric with initial data. func (m *metricSystemCPUTime) init() { - m.data.SetName("system.cpu.time") - m.data.SetDescription("Monotonic cumulative sum int metric enabled by default.") - m.data.SetUnit("s") + m.data.SetName(metricSemConvSystemCPUTime.Name()) + m.data.SetDescription(metricSemConvSystemCPUTime.Description()) + m.data.SetUnit(metricSemConvSystemCPUTime.Unit()) m.data.SetEmptySum() m.data.Sum().SetIsMonotonic(true) m.data.Sum().SetAggregationTemporality(pmetric.AggregationTemporalityCumulative) diff --git a/cmd/mdatagen/internal/samplescraper/internal/metadata/generated_metrics_test.go b/cmd/mdatagen/internal/samplescraper/internal/metadata/generated_metrics_test.go index 83eadc6472f8..f6136f1de9a9 100644 --- a/cmd/mdatagen/internal/samplescraper/internal/metadata/generated_metrics_test.go +++ b/cmd/mdatagen/internal/samplescraper/internal/metadata/generated_metrics_test.go @@ -512,7 +512,7 @@ func TestMetricsBuilder(t *testing.T) { validatedMetrics["system.cpu.time"] = true assert.Equal(t, pmetric.MetricTypeSum, ms.At(i).Type()) assert.Equal(t, 1, ms.At(i).Sum().DataPoints().Len()) - assert.Equal(t, "Monotonic cumulative sum int metric enabled by default.", ms.At(i).Description()) + assert.Equal(t, metricSemConvSystemCPUTime.Description(), ms.At(i).Description()) assert.Equal(t, "s", ms.At(i).Unit()) assert.True(t, ms.At(i).Sum().IsMonotonic()) assert.Equal(t, pmetric.AggregationTemporalityCumulative, ms.At(i).Sum().AggregationTemporality()) @@ -526,7 +526,7 @@ func TestMetricsBuilder(t *testing.T) { validatedMetrics["system.cpu.time"] = true assert.Equal(t, pmetric.MetricTypeSum, ms.At(i).Type()) assert.Equal(t, 1, ms.At(i).Sum().DataPoints().Len()) - assert.Equal(t, "Monotonic cumulative sum int metric enabled by default.", ms.At(i).Description()) + assert.Equal(t, metricSemConvSystemCPUTime.Description(), ms.At(i).Description()) assert.Equal(t, "s", ms.At(i).Unit()) assert.True(t, ms.At(i).Sum().IsMonotonic()) assert.Equal(t, pmetric.AggregationTemporalityCumulative, ms.At(i).Sum().AggregationTemporality()) @@ -550,3 +550,13 @@ func TestMetricsBuilder(t *testing.T) { }) } } + +func TestAttributeEnumAttrStringInvalid(t *testing.T) { + assert.Equal(t, "", AttributeEnumAttr(999).String()) +} + +func TestMapAttributeEnumAttr(t *testing.T) { + for str, val := range MapAttributeEnumAttr { + assert.Equal(t, str, val.String()) + } +} diff --git a/cmd/mdatagen/internal/templates/metrics.go.tmpl b/cmd/mdatagen/internal/templates/metrics.go.tmpl index 7efe39ed1b47..15300300201f 100644 --- a/cmd/mdatagen/internal/templates/metrics.go.tmpl +++ b/cmd/mdatagen/internal/templates/metrics.go.tmpl @@ -23,6 +23,9 @@ import ( {{- if .SemConvVersion }} conventions "go.opentelemetry.io/otel/semconv/v{{ .SemConvVersion }}" {{- end }} + {{- range .SemConvImports }} + {{ .Alias }} "go.opentelemetry.io/otel/semconv/v{{ $.SemConvVersion }}/{{ .Package }}" + {{- end }} {{ if .ResourceAttributes -}} "go.opentelemetry.io/collector/filter" {{- end }} @@ -71,11 +74,15 @@ var MapAttribute{{ $name.Render }} = map[string]Attribute{{ $name.Render }}{ {{- end }} var MetricsInfo = metricsInfo{ - {{- range $name, $metric := .Metrics }} - {{ $name.Render }}: metricInfo{ - Name: "{{ $name }}", - }, - {{- end }} + {{- range $name, $metric := .Metrics }} + {{ $name.Render }}: metricInfo{ + {{- if $metric.SemanticConvention.HasSemConvType }} + Name: metricSemConv{{ $name.Render }}.Name(), + {{- else }} + Name: "{{ $name }}", + {{- end }} + }, + {{- end }} } type metricsInfo struct { @@ -118,11 +125,22 @@ type metric{{ $name.Render }} struct { {{- end }} } +{{- if $metric.SemanticConvention.HasSemConvType }} +// metricSemConv{{ $name.Render }} provides access to the semantic convention type +var metricSemConv{{ $name.Render }} = {{ $metric.SemanticConvention.Package }}.{{ $metric.SemanticConvention.Type }}{} +{{- end }} + // init fills {{ $name }} metric with initial data. func (m *metric{{ $name.Render }}) init() { + {{- if $metric.SemanticConvention.ShouldUseSemConvValues }} + m.data.SetName(metricSemConv{{ $name.Render }}.Name()) + m.data.SetDescription(metricSemConv{{ $name.Render }}.Description()) + m.data.SetUnit(metricSemConv{{ $name.Render }}.Unit()) + {{- else }} m.data.SetName("{{ $name }}") m.data.SetDescription("{{ $metric.Description }}") m.data.SetUnit("{{ $metric.Unit }}") + {{- end }} m.data.SetEmpty{{ $metric.Data.Type }}() {{- if $metric.Data.HasMonotonic }} m.data.{{ $metric.Data.Type }}().SetIsMonotonic({{ $metric.Data.Monotonic }}) diff --git a/cmd/mdatagen/internal/templates/metrics_test.go.tmpl b/cmd/mdatagen/internal/templates/metrics_test.go.tmpl index 3ff29774f660..99d9a4f13d31 100644 --- a/cmd/mdatagen/internal/templates/metrics_test.go.tmpl +++ b/cmd/mdatagen/internal/templates/metrics_test.go.tmpl @@ -23,9 +23,9 @@ const ( testDataSetDefault testDataSet = iota testDataSetAll testDataSetNone - {{- if $reag }} - testDataSetReag - {{- end }} + {{- if $reag }} + testDataSetReag + {{- end }} ) func TestMetricsBuilder(t *testing.T) { @@ -43,13 +43,13 @@ func TestMetricsBuilder(t *testing.T) { metricsSet: testDataSetAll, resAttrsSet: testDataSetAll, }, - {{- if $reag }} - { - name: "reaggregate_set", - metricsSet: testDataSetReag, - resAttrsSet: testDataSetReag, - }, - {{- end }} + {{- if $reag }} + { + name: "reaggregate_set", + metricsSet: testDataSetReag, + resAttrsSet: testDataSetReag, + }, + {{- end }} { name: "none_set", metricsSet: testDataSetNone, @@ -73,18 +73,18 @@ func TestMetricsBuilder(t *testing.T) { start := pcommon.Timestamp(1_000_000_000) ts := pcommon.Timestamp(1_000_001_000) observedZapCore, observedLogs := observer.New(zap.WarnLevel) - {{- if or isReceiver isScraper isConnector }} - settings := {{ .Status.Class }}test.NewNopSettings({{ .Status.Class }}test.NopType) - {{- end }} + {{- if or isReceiver isScraper isConnector }} + settings := {{ .Status.Class }}test.NewNopSettings({{ .Status.Class }}test.NopType) + {{- end }} settings.Logger = zap.New(observedZapCore) mb := NewMetricsBuilder(loadMetricsBuilderConfig(t, tt.name), settings, WithStartTime(start)) - {{- if $reag }} - aggMap := make(map[string]string) // contains the aggregation strategies for each metric name - {{- range $name, $_ := .Metrics }} - aggMap["{{ $name.Render }}"] = mb.metric{{ $name.Render }}.config.AggregationStrategy - {{- end }} - {{- end }} + {{- if $reag }} + aggMap := make(map[string]string) // contains the aggregation strategies for each metric name + {{- range $name, $_ := .Metrics }} + aggMap["{{ $name.Render }}"] = mb.metric{{ $name.Render }}.config.AggregationStrategy + {{- end }} + {{- end }} expectedWarnings := 0 {{- range $name, $metric := .Metrics }} @@ -128,13 +128,13 @@ func TestMetricsBuilder(t *testing.T) { {{- end }} {{- end }} - {{- if $reag }} - if tt.metricsSet != testDataSetReag { - assert.Equal(t, expectedWarnings, observedLogs.Len()) - } - {{- else }} - assert.Equal(t, expectedWarnings, observedLogs.Len()) - {{- end }} + {{- if $reag }} + if tt.metricsSet != testDataSetReag { + assert.Equal(t, expectedWarnings, observedLogs.Len()) + } + {{- else }} + assert.Equal(t, expectedWarnings, observedLogs.Len()) + {{- end }} defaultMetricsCount := 0 allMetricsCount := 0 @@ -225,7 +225,11 @@ func TestMetricsBuilder(t *testing.T) { validatedMetrics["{{ $name }}"] = true assert.Equal(t, pmetric.MetricType{{ $metric.Data.Type }}, ms.At(i).Type()) assert.Equal(t, 1, ms.At(i).{{ $metric.Data.Type }}().DataPoints().Len()) + {{- if $metric.SemanticConvention.ShouldUseSemConvValues }} + assert.Equal(t, metricSemConv{{ $name.Render }}.Description(), ms.At(i).Description()) + {{- else }} assert.Equal(t, "{{ $metric.Description }}", ms.At(i).Description()) + {{- end }} {{- if len $metric.Unit}} assert.Equal(t, "{{ $metric.Unit }}", ms.At(i).Unit()) {{- else }} @@ -265,7 +269,11 @@ func TestMetricsBuilder(t *testing.T) { validatedMetrics["{{ $name }}"] = true assert.Equal(t, pmetric.MetricType{{ $metric.Data.Type }}, ms.At(i).Type()) assert.Equal(t, 1, ms.At(i).{{ $metric.Data.Type }}().DataPoints().Len()) + {{- if $metric.SemanticConvention.ShouldUseSemConvValues }} + assert.Equal(t, metricSemConv{{ $name.Render }}.Description(), ms.At(i).Description()) + {{- else }} assert.Equal(t, "{{ $metric.Description }}", ms.At(i).Description()) + {{- end }} {{- if len $metric.Unit}} assert.Equal(t, "{{ $metric.Unit }}", ms.At(i).Unit()) {{- else }} @@ -319,11 +327,15 @@ func TestMetricsBuilder(t *testing.T) { {{- end }} } {{- else }} - assert.False(t, validatedMetrics["{{ $name }}"], "Found a duplicate in the metrics slice: {{ $name }}") + assert.False(t, validatedMetrics["{{ $name }}"], "Found a duplicate in the metrics slice: {{ $name }}") validatedMetrics["{{ $name }}"] = true assert.Equal(t, pmetric.MetricType{{ $metric.Data.Type }}, ms.At(i).Type()) assert.Equal(t, 1, ms.At(i).{{ $metric.Data.Type }}().DataPoints().Len()) + {{- if $metric.SemanticConvention.ShouldUseSemConvValues }} + assert.Equal(t, metricSemConv{{ $name.Render }}.Description(), ms.At(i).Description()) + {{- else }} assert.Equal(t, "{{ $metric.Description }}", ms.At(i).Description()) + {{- end }} {{- if len $metric.Unit}} assert.Equal(t, "{{ $metric.Unit }}", ms.At(i).Unit()) {{- else }} @@ -364,3 +376,17 @@ func TestMetricsBuilder(t *testing.T) { }) } } + +{{ range $name, $info := .Attributes }} +{{- if $info.Enum }} +func TestAttribute{{ $name.Render }}StringInvalid(t *testing.T) { + assert.Equal(t, "", Attribute{{ $name.Render }}(999).String()) +} + +func TestMapAttribute{{ $name.Render }}(t *testing.T) { + for str, val := range MapAttribute{{ $name.Render }} { + assert.Equal(t, str, val.String()) + } +} +{{- end }} +{{ end }} diff --git a/cmd/mdatagen/internal/testdata/invalid_metric_semconvref_old_version.yaml b/cmd/mdatagen/internal/testdata/invalid_metric_semconvref_old_version.yaml new file mode 100644 index 000000000000..7570243e9ab3 --- /dev/null +++ b/cmd/mdatagen/internal/testdata/invalid_metric_semconvref_old_version.yaml @@ -0,0 +1,28 @@ +type: metricreceiver + +status: + class: receiver + stability: + development: [logs] + beta: [traces] + stable: [metrics] + distributions: [contrib] + warnings: + - Any additional information that should be brought to the consumer's attention + +sem_conv_version: 1.9.0 + +metrics: + default.metric: + enabled: true + description: Monotonic cumulative sum int metric enabled by default. + extended_documentation: The metric will be become optional soon. + stability: development + semantic_convention: + ref: https://github.com/open-telemetry/semantic-conventions/blob/v1.9.0/docs/system/system-metrics.md#metric-systemcputime + unit: s + sum: + value_type: int + monotonic: true + aggregation_temporality: cumulative + diff --git a/cmd/mdatagen/metadata-schema.yaml b/cmd/mdatagen/metadata-schema.yaml index df0fc0026c8f..ef158e3f4e31 100644 --- a/cmd/mdatagen/metadata-schema.yaml +++ b/cmd/mdatagen/metadata-schema.yaml @@ -167,7 +167,9 @@ metrics: since: # Required: migration note note: - # Optional: the reference to a semantic convention + # Optional: the reference to a semantic convention. + # When ref is provided, package and type are auto-inferred from the metric name + # (e.g. "system.cpu.time" → package "systemconv", type "CPUTime"). semantic_convention: ref: