diff --git a/tools/generate-module-dependencies/README.md b/tools/generate-module-dependencies/README.md index abf0d855942..e5143400e08 100644 --- a/tools/generate-module-dependencies/README.md +++ b/tools/generate-module-dependencies/README.md @@ -7,8 +7,8 @@ A small utility to keep Go module replace directives consistent across the repos - Reads dependency definitions from the project-level `dependency-replacements.yaml`. - Renders the list of `replace` directives using a template. -- Injects the rendered block into target files (e.g., `go.mod`) between well-known markers. -- Runs `go mod tidy` for affected modules. +- Injects the rendered block into target files (e.g., `go.mod` or OCB builder config YAML files) between well-known markers. +- Runs `go mod tidy` for affected modules with `file_type` of mod Generated blocks are wrapped with: ``` @@ -19,7 +19,14 @@ BEGIN GENERATED REPLACES - DO NOT EDIT MANUALLY ... END GENERATED REPLACES changed manually within these markers will be overwritten during the next run. Please note that local replacement directives (ie pointing a dependency to a local module) are _not_ meant to be included -in `dependency-replacements.yaml`. These must be included separately in `go.mod` files, outside of the template boundaries +in `dependency-replacements.yaml`. These must be included separately in `go.mod` files, outside of the template boundaries. + +## Supported File Types + +The tool supports two file types: + +- **`mod`**: Go module files (`go.mod`) +- **`ocb`**: OpenTelemetry Collector Builder (OCB) config YAML files ## Usage @@ -31,10 +38,31 @@ in `dependency-replacements.yaml`. These must be included separately in `go.mod` All inputs come from `dependency-replacements.yaml`, which defines: - Modules to update (name, path, file_type). + - `file_type` can be `mod` for `go.mod` files or `ocb` for OCB builder config YAML files - Replace entries (dependency, replacement, optional comment). Comments are normalized (single-line) and included above the corresponding `replace` directive in generated output. +### Example `dependency-replacements.yaml` + +```yaml +modules: + - name: main + path: go.mod + file_type: mod + - name: collector + path: collector/builder-config.yaml + file_type: ocb + +replaces: + - dependency: example.com/package + replacement: example.com/fork v1.0.0 + comment: Test replace for example.com/package + - dependency: github.com/test/dependency + replacement: github.com/test/fork v1.0.0 + comment: Another test replace +``` + ## Troubleshooting - If a start marker exists without an end marker (or vice versa), generation fails—ensure both markers are present or absent together. diff --git a/tools/generate-module-dependencies/cmd/sync-mod.go b/tools/generate-module-dependencies/cmd/sync-mod.go index 9118e54e4a5..b7685598161 100644 --- a/tools/generate-module-dependencies/cmd/sync-mod.go +++ b/tools/generate-module-dependencies/cmd/sync-mod.go @@ -9,27 +9,34 @@ import ( "github.com/spf13/cobra" ) -var generateAndApplyReplaces = &cobra.Command{ - Use: "generate", - Short: "Generates replace directives as specified in the input dependency-replacements.yaml", - Run: func(cmd *cobra.Command, args []string) { - pathToYaml := cmd.Flag("dependency-yaml").Value.String() - pathToRoot := cmd.Flag("project-root").Value.String() - - fileHelper, err := helpers.NewFileHelper(pathToYaml, pathToRoot) - if err != nil { - log.Fatalf("Failed to create file helper: %v", err) - } - - projectReplaces, err := fileHelper.LoadProjectReplaces() - if err != nil { - log.Fatalf("Failed to load project replaces: %v", err) - } - - modByReplaceStr := internal.GenerateReplaces(fileHelper, projectReplaces) - internal.ApplyReplaces(fileHelper, projectReplaces, modByReplaceStr) - internal.TidyModules(fileHelper, projectReplaces) - }, +func newGenerateCommand() *cobra.Command { + generateAndApplyReplaces := &cobra.Command{ + Use: "generate", + Short: "Generates replace directives as specified in the input dependency-replacements.yaml", + Run: func(cmd *cobra.Command, args []string) { + pathToYaml := cmd.Flag("dependency-yaml").Value.String() + pathToRoot := cmd.Flag("project-root").Value.String() + + fileHelper, err := helpers.NewFileHelper(pathToYaml, pathToRoot) + if err != nil { + log.Fatalf("Failed to create file helper: %v", err) + } + + projectReplaces, err := fileHelper.LoadProjectReplaces() + if err != nil { + log.Fatalf("Failed to load project replaces: %v", err) + } + + modByReplaceStr := internal.GenerateReplaces(fileHelper, projectReplaces) + internal.ApplyReplaces(fileHelper, projectReplaces, modByReplaceStr) + internal.TidyModules(fileHelper, projectReplaces) + }, + } + + generateAndApplyReplaces.Flags().String("dependency-yaml", "", "Relative path to the dependency-replacements.yaml file") + generateAndApplyReplaces.Flags().String("project-root", "", "Relative path to the project root") + + return generateAndApplyReplaces } func Execute() { @@ -44,9 +51,7 @@ func NewRootCommand() *cobra.Command { CompletionOptions: cobra.CompletionOptions{DisableDefaultCmd: true}, } - rootCmd.AddCommand(generateAndApplyReplaces) - generateAndApplyReplaces.Flags().String("dependency-yaml", "", "Relative path to the dependency-replacements.yaml file") - generateAndApplyReplaces.Flags().String("project-root", "", "Relative path to the project root") + rootCmd.AddCommand(newGenerateCommand()) return rootCmd } diff --git a/tools/generate-module-dependencies/e2e_test.go b/tools/generate-module-dependencies/e2e_test.go index 6a543405c8e..0c4d308aa17 100644 --- a/tools/generate-module-dependencies/e2e_test.go +++ b/tools/generate-module-dependencies/e2e_test.go @@ -14,18 +14,7 @@ type testCase struct { testdataDir string } -var testCases = []testCase{ - { - name: "Basic", - testdataDir: "basic-mod", - }, - { - name: "UpdateExisting", - testdataDir: "update-existing-mod", - }, -} - -func TestE2E(t *testing.T) { +func TestE2EMod(t *testing.T) { currentDir, err := os.Getwd() if err != nil { t.Fatalf("Failed to get current working directory: %v", err) @@ -33,6 +22,17 @@ func TestE2E(t *testing.T) { command := cmd.NewRootCommand() + var testCases = []testCase{ + { + name: "Basic", + testdataDir: "basic-mod", + }, + { + name: "UpdateExisting", + testdataDir: "update-existing-mod", + }, + } + for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { testdataDir := filepath.Join(currentDir, "testdata", tc.testdataDir) @@ -77,3 +77,67 @@ func TestE2E(t *testing.T) { }) } } + +func TestE2EOCB(t *testing.T) { + currentDir, err := os.Getwd() + if err != nil { + t.Fatalf("Failed to get current working directory: %v", err) + } + + command := cmd.NewRootCommand() + + var testCases = []testCase{ + { + name: "Basic", + testdataDir: "basic-ocb", + }, + { + name: "UpdateExisting", + testdataDir: "update-existing-ocb", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + testdataDir := filepath.Join(currentDir, "testdata", tc.testdataDir) + builderYamlPath := filepath.Join(testdataDir, "test-builder-config.yaml") + expectedPath := filepath.Join(testdataDir, "test-builder-config-expected.yaml") + dependencyYaml := filepath.Join("testdata", tc.testdataDir, "dependency-replacements-test.yaml") + projectRoot := filepath.Join("testdata", tc.testdataDir) + + originalYaml, err := os.ReadFile(builderYamlPath) + if err != nil { + t.Fatalf("Failed to read original builder yaml: %v", err) + } + + // Restore the original builder yaml after the test + defer func() { + if err := os.WriteFile(builderYamlPath, originalYaml, 0644); err != nil { + t.Errorf("Failed to restore original builder yaml: %v", err) + } + }() + + command.SetArgs([]string{"generate", "--dependency-yaml", dependencyYaml, "--project-root", projectRoot}) + err = command.Execute() + if err != nil { + t.Fatalf("Failed to run command: %v", err) + } + + expectedContent, err := os.ReadFile(expectedPath) + if err != nil { + t.Fatalf("Failed to read expected builder yaml: %v", err) + } + expectedYaml := strings.TrimSpace(string(expectedContent)) + + actualContent, err := os.ReadFile(builderYamlPath) + if err != nil { + t.Fatalf("Failed to read actual builder yaml: %v", err) + } + actualYaml := strings.TrimSpace(string(actualContent)) + + if actualYaml != expectedYaml { + t.Errorf("builder yaml content mismatch.\nExpected:\n%s\n\nActual:\n%s", expectedYaml, actualYaml) + } + }) + } +} diff --git a/tools/generate-module-dependencies/internal/apply-replaces.go b/tools/generate-module-dependencies/internal/apply-replaces.go index dd5e0c71013..6d3260f9944 100644 --- a/tools/generate-module-dependencies/internal/apply-replaces.go +++ b/tools/generate-module-dependencies/internal/apply-replaces.go @@ -49,8 +49,10 @@ func getCommentMarker(fileType types.FileType) (string, error) { switch fileType { case types.FileTypeMod: return "//", nil + case types.FileTypeOCB: + return "#", nil default: - return "", fmt.Errorf("unknown file_type %q", fileType) + return "", fmt.Errorf("unknown file_type %q (expected %q or %q)", fileType, types.FileTypeMod, types.FileTypeOCB) } } @@ -67,30 +69,37 @@ func getMarkers(fileType types.FileType) (startMarker, endMarker string, err err // Upserts the generated block using the markers, or lack thereof, as a guide func upsertGeneratedBlock(targetContent, replacement, startMarker, endMarker string) (string, error) { - startIdx := strings.Index(targetContent, startMarker) - startFound := startIdx != -1 + lines := strings.Split(targetContent, "\n") - if !startFound { - // No start marker: if the end marker exists anywhere, it's invalid. - if strings.Contains(targetContent, endMarker) { - return "", fmt.Errorf("found end marker without start marker") - } + startLineIdx := -1 + endLineIdx := -1 - // Neither start not end marker found, append to the end of the file - targetContent = strings.TrimRight(targetContent, "\n") - return targetContent + "\n" + replacement, nil + for i, line := range lines { + if strings.Contains(line, startMarker) { + startLineIdx = i + } + if strings.Contains(line, endMarker) { + endLineIdx = i + } } - searchFrom := startIdx + len(startMarker) - endRel := strings.Index(targetContent[searchFrom:], endMarker) - if endRel == -1 { - // Start marker exists without an end marker, which is invalid + if startLineIdx == -1 && endLineIdx != -1 { + return "", fmt.Errorf("found end marker without start marker") + } + if startLineIdx != -1 && endLineIdx == -1 { return "", fmt.Errorf("found start marker without end marker") } - endIdx := searchFrom + endRel - endOfMarker := endIdx + len(endMarker) + // If no markers are found we optimistically append to the end of the file + if startLineIdx == -1 { + trimmed := strings.TrimRight(targetContent, "\n") + return trimmed + "\n" + replacement, nil + } + + // Splice the replacement lines between the startLine and endLine + replacementLines := strings.Split(replacement, "\n") + newLines := append(lines[:startLineIdx], replacementLines...) + newLines = append(newLines, lines[endLineIdx+1:]...) - // Replace [startIdx, endOfMarker) with replacement. - return targetContent[:startIdx] + replacement + targetContent[endOfMarker:], nil + return strings.Join(newLines, "\n"), nil } diff --git a/tools/generate-module-dependencies/internal/helpers/files.go b/tools/generate-module-dependencies/internal/helpers/files.go index 99f9ed20f97..eb2b8833624 100644 --- a/tools/generate-module-dependencies/internal/helpers/files.go +++ b/tools/generate-module-dependencies/internal/helpers/files.go @@ -50,8 +50,10 @@ func (d *FileHelper) TemplatePath(fileType types.FileType) (string, error) { switch fileType { case types.FileTypeMod: templateName = "replaces-mod.tpl" + case types.FileTypeOCB: + templateName = "replaces-ocb.tpl" default: - err = fmt.Errorf("Unknown file_type %q (expected %q)", fileType, types.FileTypeMod) + err = fmt.Errorf("unknown file_type %q (expected %q or %q)", fileType, types.FileTypeMod, types.FileTypeOCB) } return filepath.Join(d.ScriptDir, templateName), err } diff --git a/tools/generate-module-dependencies/internal/tidy-modules.go b/tools/generate-module-dependencies/internal/tidy-modules.go index e7290769c76..2577eb68520 100644 --- a/tools/generate-module-dependencies/internal/tidy-modules.go +++ b/tools/generate-module-dependencies/internal/tidy-modules.go @@ -11,6 +11,10 @@ import ( func TidyModules(fileHelper *helpers.FileHelper, projectReplaces *types.ProjectReplaces) { for _, module := range projectReplaces.Modules { + if module.FileType != types.FileTypeMod { + continue + } + if err := runGoModTidy(fileHelper, module); err != nil { log.Fatalf("Failed to run go mod tidy for module %q: %v", module.Name, err) } diff --git a/tools/generate-module-dependencies/internal/types/types.go b/tools/generate-module-dependencies/internal/types/types.go index 0a1fa9432e3..d6f42f0294e 100644 --- a/tools/generate-module-dependencies/internal/types/types.go +++ b/tools/generate-module-dependencies/internal/types/types.go @@ -10,6 +10,7 @@ type FileType string const ( FileTypeMod FileType = "mod" + FileTypeOCB FileType = "ocb" ) func (ft FileType) String() string { @@ -26,8 +27,11 @@ func (ft *FileType) UnmarshalYAML(value *yaml.Node) error { case FileTypeMod: *ft = FileTypeMod return nil + case FileTypeOCB: + *ft = FileTypeOCB + return nil default: - return fmt.Errorf("invalid Module.file_type %q (expected %q)", s, FileTypeMod) + return fmt.Errorf("invalid Module.file_type %q (expected %q or %q)", s, FileTypeMod, FileTypeOCB) } } diff --git a/tools/generate-module-dependencies/replaces-ocb.tpl b/tools/generate-module-dependencies/replaces-ocb.tpl new file mode 100644 index 00000000000..fa06914b979 --- /dev/null +++ b/tools/generate-module-dependencies/replaces-ocb.tpl @@ -0,0 +1,8 @@ + # BEGIN GENERATED REPLACES - DO NOT EDIT MANUALLY +{{- range . }} +{{- if .Comment }} + # {{ .Comment }} +{{- end }} + - {{ .Dependency }} => {{ .Replacement }} +{{- end }} + # END GENERATED REPLACES \ No newline at end of file diff --git a/tools/generate-module-dependencies/testdata/basic-ocb/dependency-replacements-test.yaml b/tools/generate-module-dependencies/testdata/basic-ocb/dependency-replacements-test.yaml new file mode 100644 index 00000000000..4dd038ad99b --- /dev/null +++ b/tools/generate-module-dependencies/testdata/basic-ocb/dependency-replacements-test.yaml @@ -0,0 +1,13 @@ +modules: + - name: test-collector + path: test-builder-config.yaml + file_type: ocb + +replaces: + - comment: Test replace for example.com/package + dependency: example.com/package + replacement: example.com/fork v1.0.0 + + - comment: Another test replace + dependency: github.com/test/dependency + replacement: github.com/test/fork v1.0.0 \ No newline at end of file diff --git a/tools/generate-module-dependencies/testdata/basic-ocb/test-builder-config-expected.yaml b/tools/generate-module-dependencies/testdata/basic-ocb/test-builder-config-expected.yaml new file mode 100644 index 00000000000..c82879c7581 --- /dev/null +++ b/tools/generate-module-dependencies/testdata/basic-ocb/test-builder-config-expected.yaml @@ -0,0 +1,24 @@ +dist: + module: github.com/test + name: test + description: test distribution + version: v1.0.0 + +extensions: + - gomod: go.opentelemetry.io/collector/extension/zpagesextension v0.139.0 + +exporters: + - gomod: go.opentelemetry.io/collector/exporter/debugexporter v0.139.0 + +receivers: + - gomod: go.opentelemetry.io/collector/receiver/otlpreceiver v0.139.0 + +replaces: + # local replacements + - github.com/some-dependencys/some-dep => ./some-dep + # BEGIN GENERATED REPLACES - DO NOT EDIT MANUALLY + # Test replace for example.com/package + - example.com/package => example.com/fork v1.0.0 + # Another test replace + - github.com/test/dependency => github.com/test/fork v1.0.0 + # END GENERATED REPLACES \ No newline at end of file diff --git a/tools/generate-module-dependencies/testdata/basic-ocb/test-builder-config.yaml b/tools/generate-module-dependencies/testdata/basic-ocb/test-builder-config.yaml new file mode 100644 index 00000000000..1bff7b713c8 --- /dev/null +++ b/tools/generate-module-dependencies/testdata/basic-ocb/test-builder-config.yaml @@ -0,0 +1,21 @@ +dist: + module: github.com/test + name: test + description: test distribution + version: v1.0.0 + +extensions: + - gomod: go.opentelemetry.io/collector/extension/zpagesextension v0.139.0 + +exporters: + - gomod: go.opentelemetry.io/collector/exporter/debugexporter v0.139.0 + +receivers: + - gomod: go.opentelemetry.io/collector/receiver/otlpreceiver v0.139.0 + +replaces: + # local replacements + - github.com/some-dependencys/some-dep => ./some-dep + # BEGIN GENERATED REPLACES - DO NOT EDIT MANUALLY + + # END GENERATED REPLACES \ No newline at end of file diff --git a/tools/generate-module-dependencies/testdata/update-existing-ocb/dependency-replacements-test.yaml b/tools/generate-module-dependencies/testdata/update-existing-ocb/dependency-replacements-test.yaml new file mode 100644 index 00000000000..4dd038ad99b --- /dev/null +++ b/tools/generate-module-dependencies/testdata/update-existing-ocb/dependency-replacements-test.yaml @@ -0,0 +1,13 @@ +modules: + - name: test-collector + path: test-builder-config.yaml + file_type: ocb + +replaces: + - comment: Test replace for example.com/package + dependency: example.com/package + replacement: example.com/fork v1.0.0 + + - comment: Another test replace + dependency: github.com/test/dependency + replacement: github.com/test/fork v1.0.0 \ No newline at end of file diff --git a/tools/generate-module-dependencies/testdata/update-existing-ocb/test-builder-config-expected.yaml b/tools/generate-module-dependencies/testdata/update-existing-ocb/test-builder-config-expected.yaml new file mode 100644 index 00000000000..c82879c7581 --- /dev/null +++ b/tools/generate-module-dependencies/testdata/update-existing-ocb/test-builder-config-expected.yaml @@ -0,0 +1,24 @@ +dist: + module: github.com/test + name: test + description: test distribution + version: v1.0.0 + +extensions: + - gomod: go.opentelemetry.io/collector/extension/zpagesextension v0.139.0 + +exporters: + - gomod: go.opentelemetry.io/collector/exporter/debugexporter v0.139.0 + +receivers: + - gomod: go.opentelemetry.io/collector/receiver/otlpreceiver v0.139.0 + +replaces: + # local replacements + - github.com/some-dependencys/some-dep => ./some-dep + # BEGIN GENERATED REPLACES - DO NOT EDIT MANUALLY + # Test replace for example.com/package + - example.com/package => example.com/fork v1.0.0 + # Another test replace + - github.com/test/dependency => github.com/test/fork v1.0.0 + # END GENERATED REPLACES \ No newline at end of file diff --git a/tools/generate-module-dependencies/testdata/update-existing-ocb/test-builder-config.yaml b/tools/generate-module-dependencies/testdata/update-existing-ocb/test-builder-config.yaml new file mode 100644 index 00000000000..e39b74eddaa --- /dev/null +++ b/tools/generate-module-dependencies/testdata/update-existing-ocb/test-builder-config.yaml @@ -0,0 +1,22 @@ +dist: + module: github.com/test + name: test + description: test distribution + version: v1.0.0 + +extensions: + - gomod: go.opentelemetry.io/collector/extension/zpagesextension v0.139.0 + +exporters: + - gomod: go.opentelemetry.io/collector/exporter/debugexporter v0.139.0 + +receivers: + - gomod: go.opentelemetry.io/collector/receiver/otlpreceiver v0.139.0 + +replaces: + # local replacements + - github.com/some-dependencys/some-dep => ./some-dep + # BEGIN GENERATED REPLACES - DO NOT EDIT MANUALLY + # Test replace for example.com/package + - example.com/package => example.com/fork v2.0.0 + # END GENERATED REPLACES \ No newline at end of file