diff --git a/NOTICE.txt b/NOTICE.txt index 1cb983e6906..74e659e1b6e 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -18468,6 +18468,43 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +-------------------------------------------------------------------------------- +Dependency : golang.org/x/mod +Version: v0.23.0 +Licence type (autodetected): BSD-3-Clause +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/golang.org/x/mod@v0.23.0/LICENSE: + +Copyright 2009 The Go Authors. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google LLC nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + -------------------------------------------------------------------------------- Dependency : golang.org/x/net Version: v0.37.0 @@ -104716,43 +104753,6 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. --------------------------------------------------------------------------------- -Dependency : golang.org/x/mod -Version: v0.23.0 -Licence type (autodetected): BSD-3-Clause --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/golang.org/x/mod@v0.23.0/LICENSE: - -Copyright 2009 The Go Authors. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google LLC nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -------------------------------------------------------------------------------- Dependency : golang.org/x/oauth2 Version: v0.25.0 diff --git a/dev-tools/mage/otel/deps.go b/dev-tools/mage/otel/deps.go new file mode 100644 index 00000000000..4c949949643 --- /dev/null +++ b/dev-tools/mage/otel/deps.go @@ -0,0 +1,186 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +package otel + +import ( + "fmt" + "os" + "path/filepath" + "sort" + "strings" + + "golang.org/x/mod/modfile" + "golang.org/x/mod/module" +) + +// GetOtelDependencies returns the Otel dependencies from the given go.mod. This function applies replace directives. +func GetOtelDependencies(goModPath string) (*OtelDependencies, error) { + // read go.mod + goModBytes, err := os.ReadFile(goModPath) + if err != nil { + return nil, err + } + + goModFileName := filepath.Base(goModPath) + + modFile, err := modfile.Parse(goModFileName, goModBytes, nil) + if err != nil { + return nil, err + } + + var receivers, extensions, exporters, processors, connectors []*otelDependency + // process imports + pathToDep := make(map[string]*otelDependency) + for _, req := range modFile.Require { + dependency := newOtelDependency(req) + if dependency == nil { + continue + } + pathToDep[req.Mod.Path] = dependency + + if dependency.ComponentType == "connector" { + connectors = append(connectors, dependency) + } else if dependency.ComponentType == "exporter" { + exporters = append(exporters, dependency) + } else if dependency.ComponentType == "extension" { + extensions = append(extensions, dependency) + } else if dependency.ComponentType == "processor" { + processors = append(processors, dependency) + } else if dependency.ComponentType == "receiver" { + receivers = append(receivers, dependency) + } + } + + for _, list := range [][]*otelDependency{connectors, exporters, extensions, processors, receivers} { + sort.Slice(list, func(i, j int) bool { return list[i].Name < list[j].Name }) + } + + // take care of replaces + for _, rep := range modFile.Replace { + otelDep, ok := pathToDep[rep.Old.Path] + if ok { + otelDep.applyReplace(rep) + } + } + + return &OtelDependencies{ + Connectors: connectors, + Exporters: exporters, + Extensions: extensions, + Processors: processors, + Receivers: receivers, + }, nil +} + +type otelDependency struct { + ComponentType string + Name string + Version string + Link string + req *modfile.Require +} + +func newOtelDependency(r *modfile.Require) *otelDependency { + if !strings.Contains(r.Mod.Path, "go.opentelemetry.io/") && + !strings.Contains(r.Mod.Path, "github.com/open-telemetry/") && + !strings.Contains(r.Mod.Path, "github.com/elastic/opentelemetry-collector-components/") { + return nil + } + + if r.Indirect { + return nil + } + + componentName := getOtelComponentName(r.Mod.Path) + componentType := getOtelComponentType(r.Mod.Path) + link := getOtelDependencyLink(r.Mod.Path, r.Mod.Version) + + return &otelDependency{ + ComponentType: componentType, + Name: componentName, + Version: r.Mod.Version, + Link: link, + req: r, + } +} + +func (d *otelDependency) applyReplace(rep *modfile.Replace) { + if rep == nil || rep.Old != d.req.Mod { + return + } + d.Version = rep.New.Version + d.req.Mod = rep.New + d.Link = getOtelDependencyLink(rep.New.Path, rep.New.Version) +} + +func getOtelComponentName(dependencyName string) string { + parts := strings.Split(dependencyName, "/") + return parts[len(parts)-1] +} + +func getOtelComponentType(dependencyName string) string { + if strings.Contains(dependencyName, "/connector/") { + return "connector" + } else if strings.Contains(dependencyName, "/exporter/") { + return "exporter" + } else if strings.Contains(dependencyName, "/extension/") { + return "extension" + } else if strings.Contains(dependencyName, "/processor/") { + return "processor" + } else if strings.Contains(dependencyName, "/receiver/") { + return "receiver" + } + return "" +} + +func getOtelDependencyLink(dependencyURI string, version string) string { + dependencyRepository := getDependencyRepository(dependencyURI) + dependencyPath := strings.TrimPrefix(dependencyURI, dependencyRepository+"/") + gitRevision := fmt.Sprintf("%s/%s", dependencyPath, version) + repositoryURL := getOtelRepositoryURL(dependencyURI) + // if the version is a pseudo-version pointing to a revision without a tag, we need to extract the revision + if module.IsPseudoVersion(version) { + revision, err := module.PseudoVersionRev(version) + if err == nil { // this should never return an error, as we check it earlier + gitRevision = revision + } + } + return fmt.Sprintf("https://%s/blob/%s/%s/README.md", repositoryURL, gitRevision, dependencyPath) +} + +func getDependencyRepository(dependencyURI string) string { + dependencyURIChunks := strings.Split(dependencyURI, "/") + if len(dependencyURIChunks) < 2 { + return "" + } + var dependencyRepository string + if dependencyURIChunks[0] == "go.opentelemetry.io" { + dependencyRepository = dependencyURIChunks[0] + "/" + dependencyURIChunks[1] + } else { + dependencyRepository = dependencyURIChunks[0] + "/" + dependencyURIChunks[1] + "/" + dependencyURIChunks[2] + } + return dependencyRepository +} + +func getOtelRepositoryURL(dependencyURI string) string { + if strings.HasPrefix(dependencyURI, "go.opentelemetry.io/") { + return "github.com/open-telemetry/opentelemetry-collector" + } else if strings.HasPrefix(dependencyURI, "github.com/") { + parts := strings.SplitN(dependencyURI, "/", 4) + hostPart := parts[0] + orgPart := parts[1] + repoPart := parts[2] + return fmt.Sprintf("%s/%s/%s", hostPart, orgPart, repoPart) + } + return "" +} + +type OtelDependencies struct { + Connectors []*otelDependency + Exporters []*otelDependency + Extensions []*otelDependency + Processors []*otelDependency + Receivers []*otelDependency +} diff --git a/dev-tools/mage/otel/deps_test.go b/dev-tools/mage/otel/deps_test.go new file mode 100644 index 00000000000..3dc6ef83373 --- /dev/null +++ b/dev-tools/mage/otel/deps_test.go @@ -0,0 +1,145 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +package otel + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestGetOtelDependencies(t *testing.T) { + goModContent := `module github.com/elastic/elastic-agent + +go 1.24.1 + +require ( + github.com/elastic/opentelemetry-collector-components/connector/signaltometricsconnector v0.3.0 + github.com/open-telemetry/opentelemetry-collector-contrib/connector/routingconnector v0.119.0 + github.com/open-telemetry/opentelemetry-collector-contrib/exporter/kafkaexporter v0.119.0 + github.com/open-telemetry/opentelemetry-collector-contrib/processor/transformprocessor v0.119.0 + github.com/open-telemetry/opentelemetry-collector-contrib/extension/healthcheckextension v0.119.0 + github.com/open-telemetry/opentelemetry-collector-contrib/pkg/status v0.119.0 + github.com/open-telemetry/opentelemetry-collector-contrib/receiver/kafkareceiver v0.119.0 + github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.119.0 + go.opentelemetry.io/collector/component/componentstatus v0.119.0 + go.opentelemetry.io/collector/confmap/provider/envprovider v1.25.0 + go.opentelemetry.io/collector/exporter/debugexporter v0.119.0 + go.opentelemetry.io/collector/extension/memorylimiterextension v0.119.0 + go.opentelemetry.io/collector/processor/batchprocessor v0.119.0 + go.opentelemetry.io/collector/receiver/otlpreceiver v0.119.0 + golang.org/x/crypto v0.36.0 + github.com/elastic/elastic-agent-autodiscover v0.9.0 +) +require ( + cel.dev/expr v0.19.1 // indirect + github.com/open-telemetry/opentelemetry-collector-contrib/extension/observer v0.119.0 // indirect + go.opentelemetry.io/collector v0.119.0 // indirect + go.opentelemetry.io/collector/exporter/exporterhelper/xexporterhelper v0.119.0 // indirect + go.opentelemetry.io/collector/extension/auth v0.119.0 // indirect + go.opentelemetry.io/collector/processor/processorhelper/xprocessorhelper v0.119.0 // indirect + go.opentelemetry.io/collector/receiver/xreceiver v0.119.0 // indirect +) + +replace ( + github.com/fsnotify/fsnotify => github.com/elastic/fsnotify v1.6.1-0.20240920222514-49f82bdbc9e3 + github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.119.0 => github.com/elastic/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.0.0-20250317163643-19cd4e80024f +) + +` + tempGoModFile := filepath.Join(os.TempDir(), "go.mod") + err := os.WriteFile(tempGoModFile, []byte(goModContent), 0600) + require.NoError(t, err) + t.Cleanup(func() { + removeErr := os.Remove(tempGoModFile) + assert.NoError(t, removeErr) + }) + + expected := &OtelDependencies{ + Connectors: []*otelDependency{ + { + ComponentType: "connector", + Name: "routingconnector", + Version: "v0.119.0", + Link: "https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/connector/routingconnector/v0.119.0/connector/routingconnector/README.md", + }, + { + ComponentType: "connector", + Name: "signaltometricsconnector", + Version: "v0.3.0", + Link: "https://github.com/elastic/opentelemetry-collector-components/blob/connector/signaltometricsconnector/v0.3.0/connector/signaltometricsconnector/README.md", + }, + }, + Exporters: []*otelDependency{ + { + ComponentType: "exporter", + Name: "debugexporter", + Version: "v0.119.0", + Link: "https://github.com/open-telemetry/opentelemetry-collector/blob/exporter/debugexporter/v0.119.0/exporter/debugexporter/README.md", + }, + { + ComponentType: "exporter", + Name: "kafkaexporter", + Version: "v0.119.0", + Link: "https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/exporter/kafkaexporter/v0.119.0/exporter/kafkaexporter/README.md", + }, + }, + Extensions: []*otelDependency{ + { + ComponentType: "extension", + Name: "healthcheckextension", + Version: "v0.119.0", + Link: "https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/extension/healthcheckextension/v0.119.0/extension/healthcheckextension/README.md", + }, + { + ComponentType: "extension", + Name: "memorylimiterextension", + Version: "v0.119.0", + Link: "https://github.com/open-telemetry/opentelemetry-collector/blob/extension/memorylimiterextension/v0.119.0/extension/memorylimiterextension/README.md", + }, + }, + Processors: []*otelDependency{ + { + ComponentType: "processor", + Name: "batchprocessor", + Version: "v0.119.0", + Link: "https://github.com/open-telemetry/opentelemetry-collector/blob/processor/batchprocessor/v0.119.0/processor/batchprocessor/README.md", + }, + { + ComponentType: "processor", + Name: "transformprocessor", + Version: "v0.119.0", + Link: "https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/processor/transformprocessor/v0.119.0/processor/transformprocessor/README.md", + }, + }, + Receivers: []*otelDependency{ + { + ComponentType: "receiver", + Name: "kafkareceiver", + Version: "v0.119.0", + Link: "https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/receiver/kafkareceiver/v0.119.0/receiver/kafkareceiver/README.md", + }, + { + ComponentType: "receiver", + Name: "otlpreceiver", + Version: "v0.119.0", + Link: "https://github.com/open-telemetry/opentelemetry-collector/blob/receiver/otlpreceiver/v0.119.0/receiver/otlpreceiver/README.md", + }, + { + ComponentType: "receiver", + Name: "prometheusreceiver", + Version: "v0.0.0-20250317163643-19cd4e80024f", + Link: "https://github.com/elastic/opentelemetry-collector-contrib/blob/19cd4e80024f/receiver/prometheusreceiver/README.md", + }, + }, + } + + actual, err := GetOtelDependencies(tempGoModFile) + require.NoError(t, err) + assert.EqualExportedValues(t, expected, actual) +} diff --git a/go.mod b/go.mod index e634680d95a..62629f17b8b 100644 --- a/go.mod +++ b/go.mod @@ -79,6 +79,7 @@ require ( go.uber.org/zap v1.27.0 golang.org/x/crypto v0.36.0 golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 + golang.org/x/mod v0.23.0 golang.org/x/net v0.37.0 golang.org/x/sync v0.12.0 golang.org/x/sys v0.31.0 @@ -615,7 +616,6 @@ require ( go.starlark.net v0.0.0-20230525235612-a134d8f9ddca // indirect go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/mod v0.23.0 // indirect golang.org/x/oauth2 v0.25.0 // indirect golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect gonum.org/v1/gonum v0.15.1 // indirect diff --git a/magefile.go b/magefile.go index 0970b17e9ce..6e62f250aca 100644 --- a/magefile.go +++ b/magefile.go @@ -26,13 +26,14 @@ import ( "regexp" "runtime" "slices" - "sort" "strconv" "strings" "sync" "sync/atomic" "time" + "github.com/elastic/elastic-agent/dev-tools/mage/otel" + "github.com/jedib0t/go-pretty/v6/table" "github.com/otiai10/copy" @@ -3248,7 +3249,7 @@ func (Otel) Readme() error { return fmt.Errorf("failed to parse README template: %w", err) } - data, err := getOtelDependencies() + data, err := otel.GetOtelDependencies("go.mod") if err != nil { return fmt.Errorf("Failed to get OTel dependencies: %w", err) } @@ -3270,151 +3271,6 @@ func (Otel) Readme() error { return nil } -func getOtelDependencies() (*otelDependencies, error) { - // read go.mod - readFile, err := os.Open("go.mod") - if err != nil { - return nil, err - } - defer readFile.Close() - - scanner := bufio.NewScanner(readFile) - - scanner.Split(bufio.ScanLines) - var receivers, extensions, exporters, processors, connectors []*otelDependency - // process imports - for scanner.Scan() { - l := strings.TrimSpace(scanner.Text()) - dependency := newOtelDependency(l) - if dependency == nil { - continue - } - - if dependency.ComponentType == "connector" { - connectors = append(connectors, dependency) - } else if dependency.ComponentType == "exporter" { - exporters = append(exporters, dependency) - } else if dependency.ComponentType == "extension" { - extensions = append(extensions, dependency) - } else if dependency.ComponentType == "processor" { - processors = append(processors, dependency) - } else if dependency.ComponentType == "receiver" { - receivers = append(receivers, dependency) - } - } - - for _, list := range [][]*otelDependency{connectors, exporters, extensions, processors, receivers} { - sort.Slice(list, func(i, j int) bool { return list[i].Name < list[j].Name }) - } - - return &otelDependencies{ - Connectors: connectors, - Exporters: exporters, - Extensions: extensions, - Processors: processors, - Receivers: receivers, - }, nil -} - -type otelDependency struct { - ComponentType string - Name string - Version string - Link string -} - -func newOtelDependency(l string) *otelDependency { - if !strings.Contains(l, "go.opentelemetry.io/") && - !strings.Contains(l, "github.com/open-telemetry/") && - !strings.Contains(l, "github.com/elastic/opentelemetry-collector-components/") { - return nil - } - - if strings.Contains(l, "// indirect") { - return nil - } - - chunks := strings.SplitN(l, " ", 2) - if len(chunks) != 2 { - return nil - } - dependencyURI := chunks[0] - version := chunks[1] - - componentName := getOtelComponentName(dependencyURI) - componentType := getOtelComponentType(dependencyURI) - link := getOtelDependencyLink(dependencyURI, version) - - return &otelDependency{ - ComponentType: componentType, - Name: componentName, - Version: version, - Link: link, - } -} - -func getOtelComponentName(dependencyName string) string { - parts := strings.Split(dependencyName, "/") - return parts[len(parts)-1] -} - -func getOtelComponentType(dependencyName string) string { - if strings.Contains(dependencyName, "/connector/") { - return "connector" - } else if strings.Contains(dependencyName, "/exporter/") { - return "exporter" - } else if strings.Contains(dependencyName, "/extension/") { - return "extension" - } else if strings.Contains(dependencyName, "/processor/") { - return "processor" - } else if strings.Contains(dependencyName, "/receiver/") { - return "receiver" - } - return "" -} - -func getOtelDependencyLink(dependencyURI string, version string) string { - dependencyRepository := getDependencyRepository(dependencyURI) - dependencyPath := strings.TrimPrefix(dependencyURI, dependencyRepository+"/") - repositoryURL := getOtelRepositoryURL(dependencyURI) - return fmt.Sprintf("https://%s/blob/%s/%s/%s/README.md", repositoryURL, dependencyPath, version, dependencyPath) -} - -func getDependencyRepository(dependencyURI string) string { - dependencyURIChunks := strings.Split(dependencyURI, "/") - if len(dependencyURIChunks) < 2 { - return "" - } - var dependencyRepository string - if dependencyURIChunks[0] == "go.opentelemetry.io" { - dependencyRepository = dependencyURIChunks[0] + "/" + dependencyURIChunks[1] - } else { - dependencyRepository = dependencyURIChunks[0] + "/" + dependencyURIChunks[1] + "/" + dependencyURIChunks[2] - } - return dependencyRepository -} - -func getOtelRepositoryURL(dependencyURI string) string { - if strings.HasPrefix(dependencyURI, "go.opentelemetry.io/") { - return "github.com/open-telemetry/opentelemetry-collector" - } else if strings.HasPrefix(dependencyURI, "github.com/") { - parts := strings.SplitN(dependencyURI, "/", 4) - hostPart := parts[0] - orgPart := parts[1] - repoPart := parts[2] - return fmt.Sprintf("%s/%s/%s", hostPart, orgPart, repoPart) - } - return "" -} - -type otelDependencies struct { - Connectors []*otelDependency - Exporters []*otelDependency - Extensions []*otelDependency - Processors []*otelDependency - Receivers []*otelDependency -} - type Helm mg.Namespace // RenderExamples runs the equivalent of `helm template` and `helm lint`