From 0e010044b5155487c45bd165e4b2941880cfe3c5 Mon Sep 17 00:00:00 2001 From: Amir Blum Date: Fri, 13 Dec 2024 21:50:22 +0200 Subject: [PATCH] feat: add nop destination and cleanup deprecated components (#1984) This PR: - removes the deprecated and unused logging exporter, which is replaced with the debug exporter which we do use. - removed the deprecated `ballast` extension which we don't use and is deprecated in favor of `GOMEMLIMIT` which we do use. - adds the `nop` exporter which is useful for development and testing (for example - load scenarios) - adds documentation in `CONTRIBUTING.md` regarding how to use the debug and nop destination for development. --- CONTRIBUTING.md | 18 ++++++++++++ Makefile | 9 ++++++ collector/builder-config.yaml | 3 +- collector/odigosotelcol/components.go | 9 ++---- collector/odigosotelcol/go.mod | 3 +- collector/odigosotelcol/go.sum | 6 ++-- common/config/nop.go | 40 +++++++++++++++++++++++++++ common/config/root.go | 2 +- common/dests.go | 1 + tests/debug-exporter.yaml | 14 ++++++++++ tests/nop-exporter.yaml | 14 ++++++++++ 11 files changed, 104 insertions(+), 15 deletions(-) create mode 100644 common/config/nop.go create mode 100644 tests/debug-exporter.yaml create mode 100644 tests/nop-exporter.yaml diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 55bc45cb6..d9bd15684 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -251,3 +251,21 @@ spec: ``` Once this is done, you can use the .vscode/launch.json configuration and run instrumentor local for debugging. + +## Odigos Collector Distribution + +### Debugging and Trouble Shooting + +It is sometimes necessary to look at the data flowing through the collector pipeline while debugging or troubleshooting. This can be done by adding a debug destination to the collector configuration. + +This collector will write 2 telemetry items per second to the cluster collector logs. + +```sh +make dev-debug-destination +``` + +It you want to have the pipeline but don't want to send data anywhere, use the nop destination: + +```sh +make dev-nop-destination +``` diff --git a/Makefile b/Makefile index f53b2179a..10bfb6088 100644 --- a/Makefile +++ b/Makefile @@ -237,3 +237,12 @@ dev-tests-setup: dev-tests-kind-cluster cli-build build-images load-to-kind .PHONY: dev-tests-setup-no-build dev-tests-setup-no-build: TAG := e2e-test dev-tests-setup-no-build: dev-tests-kind-cluster load-to-kind + +# Use this for debug to add a destination which only prints samples of telemetry items to the cluster gateway collector logs +.PHONY: dev-debug-destination +dev-debug-destination: + kubectl apply -f ./tests/debug-exporter.yaml + +.PHONY: dev-add-nop-destination +dev-nop-destination: + kubectl apply -f ./tests/nop-exporter.yaml diff --git a/collector/builder-config.yaml b/collector/builder-config.yaml index e3d7fd538..769d3efe4 100644 --- a/collector/builder-config.yaml +++ b/collector/builder-config.yaml @@ -8,14 +8,13 @@ dist: extensions: - gomod: go.opentelemetry.io/collector/extension/zpagesextension v0.106.0 - - gomod: go.opentelemetry.io/collector/extension/ballastextension v0.106.0 - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/extension/healthcheckextension v0.106.0 - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/extension/pprofextension v0.106.0 - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/extension/basicauthextension v0.106.0 exporters: - gomod: go.opentelemetry.io/collector/exporter/debugexporter v0.106.0 - - gomod: go.opentelemetry.io/collector/exporter/loggingexporter v0.106.0 + - gomod: go.opentelemetry.io/collector/exporter/nopexporter v0.106.0 - gomod: go.opentelemetry.io/collector/exporter/otlpexporter v0.106.0 - gomod: go.opentelemetry.io/collector/exporter/otlphttpexporter v0.106.0 - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/odigos/exporter/azureblobstorageexporter v0.106.0 diff --git a/collector/odigosotelcol/components.go b/collector/odigosotelcol/components.go index 11ffcac25..033df2631 100644 --- a/collector/odigosotelcol/components.go +++ b/collector/odigosotelcol/components.go @@ -18,7 +18,7 @@ import ( servicegraphconnector "github.com/open-telemetry/opentelemetry-collector-contrib/connector/servicegraphconnector" spanmetricsconnector "github.com/open-telemetry/opentelemetry-collector-contrib/connector/spanmetricsconnector" debugexporter "go.opentelemetry.io/collector/exporter/debugexporter" - loggingexporter "go.opentelemetry.io/collector/exporter/loggingexporter" + nopexporter "go.opentelemetry.io/collector/exporter/nopexporter" otlpexporter "go.opentelemetry.io/collector/exporter/otlpexporter" otlphttpexporter "go.opentelemetry.io/collector/exporter/otlphttpexporter" azureblobstorageexporter "github.com/open-telemetry/opentelemetry-collector-contrib/odigos/exporter/azureblobstorageexporter" @@ -61,7 +61,6 @@ import ( tencentcloudlogserviceexporter "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/tencentcloudlogserviceexporter" zipkinexporter "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/zipkinexporter" zpagesextension "go.opentelemetry.io/collector/extension/zpagesextension" - ballastextension "go.opentelemetry.io/collector/extension/ballastextension" healthcheckextension "github.com/open-telemetry/opentelemetry-collector-contrib/extension/healthcheckextension" pprofextension "github.com/open-telemetry/opentelemetry-collector-contrib/extension/pprofextension" basicauthextension "github.com/open-telemetry/opentelemetry-collector-contrib/extension/basicauthextension" @@ -104,7 +103,6 @@ func components() (otelcol.Factories, error) { factories.Extensions, err = extension.MakeFactoryMap( zpagesextension.NewFactory(), - ballastextension.NewFactory(), healthcheckextension.NewFactory(), pprofextension.NewFactory(), basicauthextension.NewFactory(), @@ -114,7 +112,6 @@ func components() (otelcol.Factories, error) { } factories.ExtensionModules = make(map[component.Type]string, len(factories.Extensions)) factories.ExtensionModules[zpagesextension.NewFactory().Type()] = "go.opentelemetry.io/collector/extension/zpagesextension v0.106.0" - factories.ExtensionModules[ballastextension.NewFactory().Type()] = "go.opentelemetry.io/collector/extension/ballastextension v0.106.0" factories.ExtensionModules[healthcheckextension.NewFactory().Type()] = "github.com/open-telemetry/opentelemetry-collector-contrib/extension/healthcheckextension v0.106.0" factories.ExtensionModules[pprofextension.NewFactory().Type()] = "github.com/open-telemetry/opentelemetry-collector-contrib/extension/pprofextension v0.106.0" factories.ExtensionModules[basicauthextension.NewFactory().Type()] = "github.com/open-telemetry/opentelemetry-collector-contrib/extension/basicauthextension v0.106.0" @@ -138,7 +135,7 @@ func components() (otelcol.Factories, error) { factories.Exporters, err = exporter.MakeFactoryMap( debugexporter.NewFactory(), - loggingexporter.NewFactory(), + nopexporter.NewFactory(), otlpexporter.NewFactory(), otlphttpexporter.NewFactory(), azureblobstorageexporter.NewFactory(), @@ -186,7 +183,7 @@ func components() (otelcol.Factories, error) { } factories.ExporterModules = make(map[component.Type]string, len(factories.Exporters)) factories.ExporterModules[debugexporter.NewFactory().Type()] = "go.opentelemetry.io/collector/exporter/debugexporter v0.106.0" - factories.ExporterModules[loggingexporter.NewFactory().Type()] = "go.opentelemetry.io/collector/exporter/loggingexporter v0.106.0" + factories.ExporterModules[nopexporter.NewFactory().Type()] = "go.opentelemetry.io/collector/exporter/nopexporter v0.106.0" factories.ExporterModules[otlpexporter.NewFactory().Type()] = "go.opentelemetry.io/collector/exporter/otlpexporter v0.106.0" factories.ExporterModules[otlphttpexporter.NewFactory().Type()] = "go.opentelemetry.io/collector/exporter/otlphttpexporter v0.106.0" factories.ExporterModules[azureblobstorageexporter.NewFactory().Type()] = "github.com/open-telemetry/opentelemetry-collector-contrib/odigos/exporter/azureblobstorageexporter v0.106.0" diff --git a/collector/odigosotelcol/go.mod b/collector/odigosotelcol/go.mod index 121e2efa2..edcd0b33b 100644 --- a/collector/odigosotelcol/go.mod +++ b/collector/odigosotelcol/go.mod @@ -90,11 +90,10 @@ require ( go.opentelemetry.io/collector/connector/forwardconnector v0.106.0 go.opentelemetry.io/collector/exporter v0.106.0 go.opentelemetry.io/collector/exporter/debugexporter v0.106.0 - go.opentelemetry.io/collector/exporter/loggingexporter v0.106.0 + go.opentelemetry.io/collector/exporter/nopexporter v0.106.0 go.opentelemetry.io/collector/exporter/otlpexporter v0.106.0 go.opentelemetry.io/collector/exporter/otlphttpexporter v0.106.0 go.opentelemetry.io/collector/extension v0.106.0 - go.opentelemetry.io/collector/extension/ballastextension v0.106.0 go.opentelemetry.io/collector/extension/zpagesextension v0.106.0 go.opentelemetry.io/collector/odigos/providers/odigosfileprovider v0.106.0 go.opentelemetry.io/collector/otelcol v0.106.0 diff --git a/collector/odigosotelcol/go.sum b/collector/odigosotelcol/go.sum index d8037b65f..b00fb2ca9 100644 --- a/collector/odigosotelcol/go.sum +++ b/collector/odigosotelcol/go.sum @@ -1648,8 +1648,8 @@ go.opentelemetry.io/collector/exporter v0.106.0 h1:ojI2uXwSNUgUespneOxFgAHybl+kq go.opentelemetry.io/collector/exporter v0.106.0/go.mod h1:XyTH1g/X0WbG1r6cSZsWGhK2Y2+HO6k2sgEJ1ygPjxE= go.opentelemetry.io/collector/exporter/debugexporter v0.106.0 h1:OzdnzTNlG9/4gPqYMSV4KQmU5ms2F0zPPJJ+SU9/L/k= go.opentelemetry.io/collector/exporter/debugexporter v0.106.0/go.mod h1:joTvRBLXb00UFxnS3cnY8WidQpRdKiR2R6VjIZ7HOkI= -go.opentelemetry.io/collector/exporter/loggingexporter v0.106.0 h1:La+sNh06mMlRye+aJd4c6w1Spk8fhGOrek6oG5lr/qQ= -go.opentelemetry.io/collector/exporter/loggingexporter v0.106.0/go.mod h1:SopnNkOdBdluhkINezabk8xD1r5DlJWJkOEOUmxAHBc= +go.opentelemetry.io/collector/exporter/nopexporter v0.106.0 h1:shTpN6E/yaM5clsMI+65KPfHhzwyUCttdFgi2tZD3AM= +go.opentelemetry.io/collector/exporter/nopexporter v0.106.0/go.mod h1:QpGkZoh1G10So5BnFKf/bY7rmHVRK5vFqn/tcGirxIw= go.opentelemetry.io/collector/exporter/otlpexporter v0.106.0 h1:o8B5PLOTbVM3Irz+UDB+ee48+DgKt8wwp1VKZwxpQY8= go.opentelemetry.io/collector/exporter/otlpexporter v0.106.0/go.mod h1:8oHHHmWStb7dNjphyQ2sppbpy70sVojxyTSkMuSpUZw= go.opentelemetry.io/collector/exporter/otlphttpexporter v0.106.0 h1:i8DD64rHYi48R+ZoyB7fQIuuO97u6EEP50k8Q+eDUw4= @@ -1658,8 +1658,6 @@ go.opentelemetry.io/collector/extension v0.106.0 h1:E/UY7fmFOMClb6qMYsOxHz3rY4LN go.opentelemetry.io/collector/extension v0.106.0/go.mod h1:e1vJ444rIPTMn/xRwapkV6oSpr07SPi6t2xrDaGrusY= go.opentelemetry.io/collector/extension/auth v0.106.0 h1:pYXfzMH5gaLyfnH0YxNRWoy18pz7hXR+Jbt1ILNIesM= go.opentelemetry.io/collector/extension/auth v0.106.0/go.mod h1:LgFJiWTR2y6eiWcYdV3zx1LVlvu+ZD0ZQsWvuFZoOCo= -go.opentelemetry.io/collector/extension/ballastextension v0.106.0 h1:IsgDMS1GHYSB+OOjebStWPQVeNqibyJhXPwV7bf0D14= -go.opentelemetry.io/collector/extension/ballastextension v0.106.0/go.mod h1:hrpipX2QOjgT91NNB/3nDwep41YzU5STLahq8sikEJ0= go.opentelemetry.io/collector/extension/zpagesextension v0.106.0 h1:GFK8oA1w2/l28y4A6YqomjFztNg16rgURndmg6LPd7g= go.opentelemetry.io/collector/extension/zpagesextension v0.106.0/go.mod h1:yj/F/qLEfpY2upkdrcCeMgDmp1E4cBxO29Uacz6xqQM= go.opentelemetry.io/collector/featuregate v1.12.0 h1:l5WbV2vMQd2bL8ubfGrbKNtZaeJRckE12CTHvRe47Tw= diff --git a/common/config/nop.go b/common/config/nop.go new file mode 100644 index 000000000..e04cc7469 --- /dev/null +++ b/common/config/nop.go @@ -0,0 +1,40 @@ +package config + +import ( + "github.com/odigos-io/odigos/common" +) + +type Nop struct{} + +func (s *Nop) DestType() common.DestinationType { + return common.NopDestinationType +} + +func (s *Nop) ModifyConfig(dest ExporterConfigurer, currentConfig *Config) error { + exporterName := "nop/" + dest.GetID() + + currentConfig.Exporters[exporterName] = GenericMap{} + + if isTracingEnabled(dest) { + tracesPipelineName := "traces/nop-" + dest.GetID() + currentConfig.Service.Pipelines[tracesPipelineName] = Pipeline{ + Exporters: []string{exporterName}, + } + } + + if isMetricsEnabled(dest) { + metricsPipelineName := "metrics/nop-" + dest.GetID() + currentConfig.Service.Pipelines[metricsPipelineName] = Pipeline{ + Exporters: []string{exporterName}, + } + } + + if isLoggingEnabled(dest) { + logsPipelineName := "logs/nop-" + dest.GetID() + currentConfig.Service.Pipelines[logsPipelineName] = Pipeline{ + Exporters: []string{exporterName}, + } + } + + return nil +} diff --git a/common/config/root.go b/common/config/root.go index 43a0ef7d7..c9c98883c 100644 --- a/common/config/root.go +++ b/common/config/root.go @@ -20,7 +20,7 @@ var availableConfigers = []Configer{ &Tempo{}, &Loki{}, &Jaeger{}, &GenericOTLP{}, &OTLPHttp{}, &Elasticsearch{}, &Quickwit{}, &Signoz{}, &Qryn{}, &OpsVerse{}, &Splunk{}, &Lightstep{}, &GoogleCloud{}, &GoogleCloudStorage{}, &Sentry{}, &AzureBlobStorage{}, &AWSS3{}, &Dynatrace{}, &Chronosphere{}, &ElasticAPM{}, &Axiom{}, &SumoLogic{}, &Coralogix{}, &Clickhouse{}, - &Causely{}, &Uptrace{}, &Debug{}, &QrynOSS{}, + &Causely{}, &Uptrace{}, &Debug{}, &QrynOSS{}, &Nop{}, } type Configer interface { diff --git a/common/dests.go b/common/dests.go index 99a017ab4..7116cb6b6 100644 --- a/common/dests.go +++ b/common/dests.go @@ -30,6 +30,7 @@ const ( LokiDestinationType DestinationType = "loki" MiddlewareDestinationType DestinationType = "middleware" NewRelicDestinationType DestinationType = "newrelic" + NopDestinationType DestinationType = "nop" OpsVerseDestinationType DestinationType = "opsverse" OtlpHttpDestinationType DestinationType = "otlphttp" PrometheusDestinationType DestinationType = "prometheus" diff --git a/tests/debug-exporter.yaml b/tests/debug-exporter.yaml new file mode 100644 index 000000000..463b0e22b --- /dev/null +++ b/tests/debug-exporter.yaml @@ -0,0 +1,14 @@ +apiVersion: odigos.io/v1alpha1 +kind: Destination +metadata: + name: debug + namespace: odigos-system +spec: + data: + VERBOSITY: detailed + destinationName: debug + signals: + - TRACES + - METRICS + - LOGS + type: debug diff --git a/tests/nop-exporter.yaml b/tests/nop-exporter.yaml new file mode 100644 index 000000000..74dc4f176 --- /dev/null +++ b/tests/nop-exporter.yaml @@ -0,0 +1,14 @@ +apiVersion: odigos.io/v1alpha1 +kind: Destination +metadata: + name: nop + namespace: odigos-system +spec: + data: + placeholder: 'here since data is required' + destinationName: nop + signals: + - TRACES + - METRICS + - LOGS + type: nop