From 33a6810c90519967389152ac9e2e6175d7709af8 Mon Sep 17 00:00:00 2001 From: Tyler Yahn Date: Mon, 1 Jul 2024 09:55:55 -0700 Subject: [PATCH] Add the minsev package (#5817) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Resolve https://github.com/open-telemetry/opentelemetry-go/issues/5078 Add the experimental `minsev` module. This module provides a log processor that thresholds all records recorded to being above a minimum severity. ### Benchmarks ``` goos: linux goarch: amd64 pkg: go.opentelemetry.io/contrib/processors/minsev cpu: Intel(R) Core(TM) i7-8550U CPU @ 1.80GHz │ new.txt │ │ sec/op │ LogProcessor/Base-8 20.53n ± 2% LogProcessor/Enabled-8 41.27n ± 5% LogProcessor/Disabled-8 22.34n ± 8% geomean 26.65n │ new.txt │ │ B/op │ LogProcessor/Base-8 0.000 ± 0% LogProcessor/Enabled-8 0.000 ± 0% LogProcessor/Disabled-8 0.000 ± 0% geomean ¹ ¹ summaries must be >0 to compute geomean │ new.txt │ │ allocs/op │ LogProcessor/Base-8 0.000 ± 0% LogProcessor/Enabled-8 0.000 ± 0% LogProcessor/Disabled-8 0.000 ± 0% geomean ¹ ¹ summaries must be >0 to compute geomean ``` --- CHANGELOG.md | 2 + CODEOWNERS | 1 + processors/minsev/go.mod | 22 ++++ processors/minsev/go.sum | 31 +++++ processors/minsev/minsev.go | 68 +++++++++++ processors/minsev/minsev_test.go | 191 +++++++++++++++++++++++++++++++ versions.yaml | 1 + 7 files changed, 316 insertions(+) create mode 100644 processors/minsev/go.mod create mode 100644 processors/minsev/go.sum create mode 100644 processors/minsev/minsev.go create mode 100644 processors/minsev/minsev_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 96efc69e083..9276b3ddd13 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,8 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm - Add log support for the autoexport package. (#5733) - Add support for disabling the old runtime metrics using the `OTEL_GO_X_DEPRECATED_RUNTIME_METRICS=false` environment variable. (#5747) - Add support for signal-specific protocols environment variables (`OTEL_EXPORTER_OTLP_TRACES_PROTOCOL`, `OTEL_EXPORTER_OTLP_LOGS_PROTOCOL`, `OTEL_EXPORTER_OTLP_METRICS_PROTOCOL`) in `go.opentelemetry.io/contrib/exporters/autoexport`. (#5816) +- The `go.opentelemetry.io/contrib/processors/minsev` module is added. + This module provides and experimental logging processor with a configurable threshold for the minimum severity records must have to be recorded. (#5817) - The `go.opentelemetry.io/contrib/processors/baggagecopy` module. This module is a replacment of `go.opentelemetry.io/contrib/processors/baggage/baggagetrace`. (#5824) diff --git a/CODEOWNERS b/CODEOWNERS index bb111d307fd..60e19adb897 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -57,6 +57,7 @@ instrumentation/net/http/otelhttp/ @open-te instrumentation/runtime/ @open-telemetry/go-approvers @MadVikingGod processors/baggage/baggagetrace @open-telemetry/go-approvers @codeboten @MikeGoldsmith +processors/minsev @open-telemetry/go-approvers @MrAlias propagators/autoprop/ @open-telemetry/go-approvers @MrAlias propagators/aws/ @open-telemetry/go-approvers @akats7 diff --git a/processors/minsev/go.mod b/processors/minsev/go.mod new file mode 100644 index 00000000000..9e0a7d70ec6 --- /dev/null +++ b/processors/minsev/go.mod @@ -0,0 +1,22 @@ +module go.opentelemetry.io/contrib/processors/minsev + +go 1.21 + +require ( + github.com/stretchr/testify v1.9.0 + go.opentelemetry.io/otel/log v0.3.0 + go.opentelemetry.io/otel/sdk/log v0.3.0 +) + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + go.opentelemetry.io/otel v1.27.0 // indirect + go.opentelemetry.io/otel/metric v1.27.0 // indirect + go.opentelemetry.io/otel/sdk v1.27.0 // indirect + go.opentelemetry.io/otel/trace v1.27.0 // indirect + golang.org/x/sys v0.20.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/processors/minsev/go.sum b/processors/minsev/go.sum new file mode 100644 index 00000000000..4dd9bb22f3b --- /dev/null +++ b/processors/minsev/go.sum @@ -0,0 +1,31 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +go.opentelemetry.io/otel v1.27.0 h1:9BZoF3yMK/O1AafMiQTVu0YDj5Ea4hPhxCs7sGva+cg= +go.opentelemetry.io/otel v1.27.0/go.mod h1:DMpAK8fzYRzs+bi3rS5REupisuqTheUlSZJ1WnZaPAQ= +go.opentelemetry.io/otel/log v0.3.0 h1:kJRFkpUFYtny37NQzL386WbznUByZx186DpEMKhEGZs= +go.opentelemetry.io/otel/log v0.3.0/go.mod h1:ziCwqZr9soYDwGNbIL+6kAvQC+ANvjgG367HVcyR/ys= +go.opentelemetry.io/otel/metric v1.27.0 h1:hvj3vdEKyeCi4YaYfNjv2NUje8FqKqUY8IlF0FxV/ik= +go.opentelemetry.io/otel/metric v1.27.0/go.mod h1:mVFgmRlhljgBiuk/MP/oKylr4hs85GZAylncepAX/ak= +go.opentelemetry.io/otel/sdk v1.27.0 h1:mlk+/Y1gLPLn84U4tI8d3GNJmGT/eXe3ZuOXN9kTWmI= +go.opentelemetry.io/otel/sdk v1.27.0/go.mod h1:Ha9vbLwJE6W86YstIywK2xFfPjbWlCuwPtMkKdz/Y4A= +go.opentelemetry.io/otel/sdk/log v0.3.0 h1:GEjJ8iftz2l+XO1GF2856r7yYVh74URiF9JMcAacr5U= +go.opentelemetry.io/otel/sdk/log v0.3.0/go.mod h1:BwCxtmux6ACLuys1wlbc0+vGBd+xytjmjajwqqIul2g= +go.opentelemetry.io/otel/trace v1.27.0 h1:IqYb813p7cmbHk0a5y6pD5JPakbVfftRXABGt5/Rscw= +go.opentelemetry.io/otel/trace v1.27.0/go.mod h1:6RiD1hkAprV4/q+yd2ln1HG9GoPx39SuvvstaLBl+l4= +golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/processors/minsev/minsev.go b/processors/minsev/minsev.go new file mode 100644 index 00000000000..24924f9a5ed --- /dev/null +++ b/processors/minsev/minsev.go @@ -0,0 +1,68 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +// Package minsev provides an [log.Processor] that will not log any record with +// a severity below a configured threshold. +package minsev // import "go.opentelemetry.io/contrib/processors/minsev" + +import ( + "context" + + api "go.opentelemetry.io/otel/log" + "go.opentelemetry.io/otel/sdk/log" +) + +// NewLogProcessor returns a new [LogProcessor] that wraps the downstream +// [log.Processor]. +// +// If downstream is nil a default No-Op [log.Processor] is used. The returned +// processor will not be enabled for nor emit any records. +func NewLogProcessor(downstream log.Processor, minimum api.Severity) *LogProcessor { + if downstream == nil { + downstream = defaultProcessor + } + return &LogProcessor{Processor: downstream, Minimum: minimum} +} + +// LogProcessor is an [log.Processor] implementation that wraps another +// [log.Processor]. It will pass-through calls to OnEmit and Enabled for +// records with severity greater than or equal to a minimum. All other method +// calls are passed to the wrapped [log.Processor]. +// +// If the wrapped [log.Processor] is nil, calls to the LogProcessor methods +// will panic. Use [NewLogProcessor] to create a new LogProcessor that ensures +// no panics. +type LogProcessor struct { + log.Processor + + Minimum api.Severity +} + +// Compile time assertion that LogProcessor implements log.Processor. +var _ log.Processor = (*LogProcessor)(nil) + +// OnEmit passes ctx and r to the [log.Processor] that p wraps if the severity +// of record is greater than or equal to p.Minimum. Otherwise, record is +// dropped. +func (p *LogProcessor) OnEmit(ctx context.Context, record log.Record) error { + if record.Severity() >= p.Minimum { + return p.Processor.OnEmit(ctx, record) + } + return nil +} + +// Enabled returns if the [log.Processor] that p wraps is enabled if the +// severity of record is greater than or equal to p.Minimum. Otherwise false is +// returned. +func (p *LogProcessor) Enabled(ctx context.Context, record log.Record) bool { + return record.Severity() >= p.Minimum && p.Processor.Enabled(ctx, record) +} + +var defaultProcessor = noopProcessor{} + +type noopProcessor struct{} + +func (p noopProcessor) OnEmit(context.Context, log.Record) error { return nil } +func (p noopProcessor) Enabled(context.Context, log.Record) bool { return false } +func (p noopProcessor) Shutdown(context.Context) error { return nil } +func (p noopProcessor) ForceFlush(context.Context) error { return nil } diff --git a/processors/minsev/minsev_test.go b/processors/minsev/minsev_test.go new file mode 100644 index 00000000000..645895a54ac --- /dev/null +++ b/processors/minsev/minsev_test.go @@ -0,0 +1,191 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package minsev + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + + api "go.opentelemetry.io/otel/log" + "go.opentelemetry.io/otel/sdk/log" +) + +var severities = []api.Severity{ + api.SeverityTrace, api.SeverityTrace1, api.SeverityTrace2, api.SeverityTrace3, api.SeverityTrace4, + api.SeverityDebug, api.SeverityDebug1, api.SeverityDebug2, api.SeverityDebug3, api.SeverityDebug4, + api.SeverityInfo, api.SeverityInfo1, api.SeverityInfo2, api.SeverityInfo3, api.SeverityInfo4, + api.SeverityWarn, api.SeverityWarn1, api.SeverityWarn2, api.SeverityWarn3, api.SeverityWarn4, + api.SeverityError, api.SeverityError1, api.SeverityError2, api.SeverityError3, api.SeverityError4, + api.SeverityFatal, api.SeverityFatal1, api.SeverityFatal2, api.SeverityFatal3, api.SeverityFatal4, +} + +type args struct { + Ctx context.Context + Record log.Record +} + +type processor struct { + ReturnErr error + + OnEmitCalls []args + EnabledCalls []args + ForceFlushCalls []context.Context + ShutdownCalls []context.Context +} + +func (p *processor) OnEmit(ctx context.Context, r log.Record) error { + p.OnEmitCalls = append(p.OnEmitCalls, args{ctx, r}) + return p.ReturnErr +} + +func (p *processor) Enabled(ctx context.Context, r log.Record) bool { + p.EnabledCalls = append(p.EnabledCalls, args{ctx, r}) + return true +} + +func (p *processor) Shutdown(ctx context.Context) error { + p.ShutdownCalls = append(p.ShutdownCalls, ctx) + return p.ReturnErr +} + +func (p *processor) ForceFlush(ctx context.Context) error { + p.ForceFlushCalls = append(p.ForceFlushCalls, ctx) + return p.ReturnErr +} + +func (p *processor) Reset() { + p.OnEmitCalls = p.OnEmitCalls[:0] + p.EnabledCalls = p.EnabledCalls[:0] + p.ShutdownCalls = p.ShutdownCalls[:0] + p.ForceFlushCalls = p.ForceFlushCalls[:0] +} + +func TestLogProcessorOnEmit(t *testing.T) { + t.Run("Passthrough", func(t *testing.T) { + wrapped := &processor{ReturnErr: assert.AnError} + + p := NewLogProcessor(wrapped, api.SeverityTrace1) + ctx := context.Background() + r := &log.Record{} + for _, sev := range severities { + r.SetSeverity(sev) + assert.ErrorIs(t, p.OnEmit(ctx, *r), assert.AnError, sev.String()) + + if assert.Lenf(t, wrapped.OnEmitCalls, 1, "Record with severity %s not passed-through", sev) { + assert.Equal(t, ctx, wrapped.OnEmitCalls[0].Ctx, sev.String()) + assert.Equal(t, *r, wrapped.OnEmitCalls[0].Record, sev.String()) + } + wrapped.Reset() + } + }) + + t.Run("Dropped", func(t *testing.T) { + wrapped := &processor{ReturnErr: assert.AnError} + + p := NewLogProcessor(wrapped, api.SeverityFatal4+1) + ctx := context.Background() + r := &log.Record{} + for _, sev := range severities { + r.SetSeverity(sev) + assert.NoError(t, p.OnEmit(ctx, *r), assert.AnError, sev.String()) + + if !assert.Lenf(t, wrapped.OnEmitCalls, 0, "Record with severity %s passed-through", sev) { + wrapped.Reset() + } + } + }) +} + +func TestLogProcessorEnabled(t *testing.T) { + t.Run("Passthrough", func(t *testing.T) { + wrapped := &processor{} + + p := NewLogProcessor(wrapped, api.SeverityTrace1) + ctx := context.Background() + r := &log.Record{} + for _, sev := range severities { + r.SetSeverity(sev) + assert.True(t, p.Enabled(ctx, *r), sev.String()) + + if assert.Lenf(t, wrapped.EnabledCalls, 1, "Record with severity %s not passed-through", sev) { + assert.Equal(t, ctx, wrapped.EnabledCalls[0].Ctx, sev.String()) + assert.Equal(t, *r, wrapped.EnabledCalls[0].Record, sev.String()) + } + wrapped.Reset() + } + }) + + t.Run("NotEnabled", func(t *testing.T) { + wrapped := &processor{} + + p := NewLogProcessor(wrapped, api.SeverityFatal4+1) + ctx := context.Background() + r := &log.Record{} + for _, sev := range severities { + r.SetSeverity(sev) + assert.False(t, p.Enabled(ctx, *r), sev.String()) + + if !assert.Lenf(t, wrapped.EnabledCalls, 0, "Record with severity %s passed-through", sev) { + wrapped.Reset() + } + } + }) +} + +func TestLogProcessorForceFlushPassthrough(t *testing.T) { + wrapped := &processor{ReturnErr: assert.AnError} + + p := NewLogProcessor(wrapped, api.SeverityTrace1) + ctx := context.Background() + assert.ErrorIs(t, p.ForceFlush(ctx), assert.AnError) + assert.Len(t, wrapped.ForceFlushCalls, 1, "ForceFlush not passed-through") +} + +func TestLogProcessorShutdownPassthrough(t *testing.T) { + wrapped := &processor{ReturnErr: assert.AnError} + + p := NewLogProcessor(wrapped, api.SeverityTrace1) + ctx := context.Background() + assert.ErrorIs(t, p.Shutdown(ctx), assert.AnError) + assert.Len(t, wrapped.ShutdownCalls, 1, "Shutdown not passed-through") +} + +func TestLogProcessorNilDownstream(t *testing.T) { + p := NewLogProcessor(nil, api.SeverityTrace1) + ctx := context.Background() + r := log.Record{} + r.SetSeverity(api.SeverityTrace1) + assert.NotPanics(t, func() { + assert.NoError(t, p.OnEmit(ctx, r)) + assert.False(t, p.Enabled(ctx, r)) + assert.NoError(t, p.ForceFlush(ctx)) + assert.NoError(t, p.Shutdown(ctx)) + }) +} + +func BenchmarkLogProcessor(b *testing.B) { + rPtr := new(log.Record) + rPtr.SetSeverity(api.SeverityTrace) + ctx, r := context.Background(), *rPtr + + run := func(p log.Processor) func(b *testing.B) { + return func(b *testing.B) { + var err error + var enabled bool + b.ReportAllocs() + for n := 0; n < b.N; n++ { + enabled = p.Enabled(ctx, r) + err = p.OnEmit(ctx, r) + } + + _, _ = err, enabled + } + } + + b.Run("Base", run(defaultProcessor)) + b.Run("Enabled", run(NewLogProcessor(nil, api.SeverityTrace))) + b.Run("Disabled", run(NewLogProcessor(nil, api.SeverityDebug))) +} diff --git a/versions.yaml b/versions.yaml index b5a77a1e81c..8f15fb3fcaa 100644 --- a/versions.yaml +++ b/versions.yaml @@ -83,6 +83,7 @@ module-sets: modules: - go.opentelemetry.io/contrib/processors/baggage/baggagetrace - go.opentelemetry.io/contrib/processors/baggagecopy + - go.opentelemetry.io/contrib/processors/minsev experimental-detectors: version: v0.0.1 modules: