Restructure repository for OTel version update

.gitignore

@@ -12,5 +12,5 @@ go.work.sum
 .vscode
 
 # Binaries from builds
-/google-otel/otelcol-google
+/google-built-opentelemetry-collector/otelcol-google
 /otelopscol/otelopscol

Makefile

@@ -26,34 +26,19 @@ presubmit: checklicense misspell lint compare-all
 # Updating OTel Components
 ##########################
 
-OTEL_VERSION ?= v0.121.0
-OTEL_CONTRIB_VERSION ?= v0.121.0
+.PHONY: update-google-otel-components update-otelopscol-components
 
-.PHONY: update-otel-components
-update-otel-components: export OTEL_VERSION := $(OTEL_VERSION)
-update-otel-components: export OTEL_CONTRIB_VERSION := $(OTEL_CONTRIB_VERSION)
-update-otel-components: update-otel-components-deps tidy-all update-mdatagen generate-components
+update-google-otel-components: SPEC_FILE := specs/google-built-opentelemetry-collector.yaml
+update-google-otel-components: COMPONENT_DIR := components/google-built-opentelemetry-collector
 
-.PHONY: update-mdatagen
-update-mdatagen:
-	go get -u go.opentelemetry.io/collector/cmd/mdatagen@$(OTEL_VERSION)
-	TOOL_LIST=go.opentelemetry.io/collector/cmd/mdatagen $(MAKE) install-tools
+update-otelopscol-components: SPEC_FILE := specs/otelopscol.yaml
+update-otelopscol-components: COMPONENT_DIR := components/otelopscol
 
-.PHONY: update-otel-components-deps
-update-otel-components-deps:
-	PATH="$(TOOLS_DIR):${PATH}" TARGET="update-components" $(MAKE) target-all-otel-components
-
-.PHONY: generate-components
-generate-components:
-	PATH="$(TOOLS_DIR):${PATH}" TARGET="generate" $(MAKE) target-all-otel-components
-
-# This target will tag the git repo using the OTel version. Eventually this may be
-# more sophisticated if we want to supply separate tags for every subcomponent. For
-# now it is pretty simply.
-.PHONY: tag-repo
-tag-repo:
-	git tag -a $(OTEL_VERSION) -m "Update to OpenTelemetry Collector version $(OTEL_VERSION)"
-	@echo "Created git tag $(OTEL_VERSION). If it looks good, push it to the remote by running: git push origin $(OTEL_VERSION)"
+update-google-otel-components update-otelopscol-components: DISTROGEN_QUERY := go run ./cmd/distrogen -spec $(SPEC_FILE) -query
+update-google-otel-components update-otelopscol-components: export OTEL_VERSION := v$(shell $(DISTROGEN_QUERY) opentelemetry_version)
+update-google-otel-components update-otelopscol-components: export OTEL_CONTRIB_VERSION := v$(shell $(DISTROGEN_QUERY) opentelemetry_contrib_version)
+update-google-otel-components update-otelopscol-components: install-tools
+	cd $(COMPONENT_DIR) && PATH="$(TOOLS_DIR):${PATH}" $(MAKE) update-components
 
 ###################
 # Distro Generation
@@ -72,7 +57,7 @@ compare-all:
 	@./internal/tools/scripts/compare.sh
 
 GEN_GOOGLE_BUILT_OTEL=$(RUN_DISTROGEN) -spec ./specs/google-built-opentelemetry-collector.yaml \
-								 -registry ./registries/operations-collector-registry.yaml \
+								 -registry ./components/google-built-opentelemetry-collector/registry.yaml \
 								 -custom_templates ./templates/google-built-opentelemetry-collector
 .PHONY: gen-google-built-otel
 gen-google-built-otel:
@@ -91,7 +76,7 @@ compare-google-built-otel:
 	@$(GEN_GOOGLE_BUILT_OTEL) -force -compare
 
 GEN_OTELOPSCOL=$(RUN_DISTROGEN) -spec ./specs/otelopscol.yaml \
-								-registry ./registries/operations-collector-registry.yaml \
+								-registry ./components/otelopscol/registry.yaml \
 								-custom_templates ./templates/otelopscol
 .PHONY: gen-otelopscol
 gen-otelopscol:
@@ -166,7 +151,6 @@ tools-dir:
 	@mkdir -p $(TOOLS_DIR)
 
 TOOL_LIST ?= github.com/google/addlicense \
-			 go.opentelemetry.io/collector/cmd/mdatagen \
 			 github.com/client9/misspell/cmd/misspell \
 			 github.com/golangci/golangci-lint/cmd/golangci-lint \
 			 golang.org/x/tools/cmd/goimports \
@@ -178,7 +162,7 @@ install-tools: tools-dir
 	GOBIN=$(TOOLS_DIR) go install \
 	$(TOOL_LIST)
 
-ADDLICENSE_IGNORES = -ignore "**/.tools/*" \
+ADDLICENSE_IGNORES = -ignore "**/.tools/**/*" \
 					-ignore "**/docs/**/*" \
 					-ignore "**/*.md" \
 					-ignore "**/testdata/*" \
@@ -210,25 +194,20 @@ misspell:
 # Utility
 #########
 
-LIST_LOCAL_MODULES = go list -f "{{ .Dir }}" -m | grep -v ".*internal/tools.*"
-INCLUDE_OTEL_COMPONENTS = grep -e ".*receiver.*" -e ".*processor.*" -e ".*extension.*" -e ".*exporter.*"
+# This target will tag the git repo using the OTel version. Eventually this may be
+# more sophisticated if we want to supply separate tags for every subcomponent. For
+# now it is pretty simply.
+.PHONY: tag-repo
+tag-repo:
+	git tag -a $(OTEL_VERSION) -m "Update to OpenTelemetry Collector version $(OTEL_VERSION)"
+	@echo "Created git tag $(OTEL_VERSION). If it looks good, push it to the remote by running: git push origin $(OTEL_VERSION)"
 
 .PHONY: target-all-modules
 target-all-modules:
 ifndef TARGET
 	@echo "No TARGET defined."
 else
-	$(LIST_LOCAL_MODULES) |\
-	GOWORK=off xargs -t -I '{}' $(MAKE) -C {} $(TARGET)
-endif
-
-.PHONY: target-all-otel-components
-target-all-otel-components:
-ifndef TARGET
-	@echo "No TARGET defined."
-else
-	$(LIST_LOCAL_MODULES) |\
-	$(INCLUDE_OTEL_COMPONENTS) |\
+	go list -f "{{ .Dir }}" -m | grep -v ".*internal/tools.*" |\
 	GOWORK=off xargs -t -I '{}' $(MAKE) -C {} $(TARGET)
 endif
 

cmd/distrogen/distribution.go

@@ -20,6 +20,8 @@ import (
 	"io/fs"
 	"os"
 	"path/filepath"
+	"reflect"
+	"strings"
 
 	"github.com/google/go-cmp/cmp"
 )
@@ -56,6 +58,35 @@ func (s *DistributionSpec) Diff(s2 *DistributionSpec) bool {
 	return diff != ""
 }
 
+var ErrQueryValueNotFound = errors.New("not found in spec")
+var ErrQueryValueInvalid = errors.New("found in spec but unsupported type")
+
+// Query will get a field from a loaded spec based on the yaml
+// field name.
+func (s *DistributionSpec) Query(field string) (string, error) {
+	v := reflect.ValueOf(s).Elem()
+	t := v.Type()
+
+	for i := 0; i < t.NumField(); i++ {
+		structField := t.Field(i)
+		yamlTag := structField.Tag.Get("yaml")
+
+		// Handle tags like "replaces,omitempty"
+		tagName := strings.Split(yamlTag, ",")[0]
+
+		if tagName == field {
+			fieldValue := v.Field(i)
+			// Convert the field value to string.
+			// This handles basic types like string, int, bool.
+			if fieldValue.IsValid() && fieldValue.CanInterface() {
+				return fmt.Sprintf("%v", fieldValue.Interface()), nil
+			}
+			return "", fmt.Errorf("field '%s': %w", field, ErrQueryValueInvalid)
+		}
+	}
+	return "", fmt.Errorf("field '%s': %w", field, ErrQueryValueNotFound)
+}
+
 // NewDistributionSpec loads the DistributionSpec from a yaml file.
 func NewDistributionSpec(path string) (*DistributionSpec, error) {
 	spec, err := yamlUnmarshalFromFile[DistributionSpec](path)
@@ -240,14 +271,14 @@ func (d *DistributionGenerator) Compare() error {
 
 	logger.Debug("comparing %s to %s", d.GeneratePath, generateDest)
 
-	generatedContent, err := getGeneratedFilesInDir(generateDest)
+	generatedContent, err := d.getGeneratedFilesInDir()
 	if err != nil {
 		return wrapExitCodeError(
 			unexpectErrExitCode,
 			fmt.Errorf("could not get generated files: %w", err),
 		)
 	}
-	existingContent, err := getGeneratedFilesInDir(d.GeneratePath)
+	existingContent, err := d.getGeneratedFilesInDir()
 	if err != nil {
 		return wrapExitCodeError(
 			unexpectErrExitCode,
@@ -283,10 +314,15 @@ func (d *DistributionGenerator) Compare() error {
 	return nil
 }
 
-func getGeneratedFilesInDir(dir string) (map[string]*generatedFile, error) {
+func (dg *DistributionGenerator) getGeneratedFilesInDir() (map[string]*generatedFile, error) {
+	wd, err := os.Getwd()
+	if err != nil {
+		return nil, fmt.Errorf("could not get working directory: %w", err)
+	}
+	dir := filepath.Join(wd, dg.GenerateDirName)
 	files := map[string]*generatedFile{}
 
-	err := filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error {
+	err = filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error {
 		if err != nil {
 			return err
 		}
@@ -295,6 +331,15 @@ func getGeneratedFilesInDir(dir string) (map[string]*generatedFile, error) {
 			return nil
 		}
 
+		// Don't include .tools directory in comparison.
+		if strings.Contains(path, "/.tools/") {
+			return nil
+		}
+
+		if d.Name() == dg.Spec.BinaryName {
+			return nil
+		}
+
 		content, err := os.ReadFile(path)
 		if err != nil {
 			return err

cmd/distrogen/distribution_test.go

@@ -117,3 +117,19 @@ func filesInDirAsSet(dir string) (map[string]bool, error) {
 	})
 	return fileSet, err
 }
+
+func TestSpecQuery(t *testing.T) {
+	otelVer := "v0.124.0"
+	spec := &DistributionSpec{
+		OpenTelemetryVersion: otelVer,
+	}
+	val, err := spec.Query("opentelemetry_version")
+	assert.NilError(t, err)
+	assert.Equal(t, val, otelVer)
+}
+
+func TestSpecQueryNotFound(t *testing.T) {
+	spec := &DistributionSpec{}
+	_, err := spec.Query("random_field_name")
+	assert.ErrorIs(t, err, ErrQueryValueNotFound)
+}

cmd/distrogen/flags.go

@@ -22,6 +22,7 @@ import (
 
 var (
 	flagSpec            = flag.String("spec", "", "The distribution specification to use")
+	flagQuery           = flag.String("query", "", "Field to query from the spec")
 	flagForce           = flag.Bool("force", false, "Force generate even if there are no differences detected")
 	flagVerbose         = flag.Bool("v", false, "Verbose output")
 	flagRegistry        = newArrayFlag("registry", "Provide additional component registries")

cmd/distrogen/main.go

@@ -47,6 +47,13 @@ func main() {
 }
 
 func run() error {
+	if *flagVerbose {
+		logLevel.Set(slog.LevelDebug)
+	}
+
+	if *flagQuery != "" {
+		return querySpec()
+	}
 	if *flagOtelConfig != "" {
 		return generateSpec()
 	}
@@ -66,14 +73,30 @@ func generateSpec() error {
 	return yamlMarshalToFile(distro, "generated_spec.yaml")
 }
 
-func generateDistribution() error {
-	specPath := *flagSpec
+func querySpec() error {
 	if *flagSpec == "" {
 		return errNoSpecFlag
 	}
 
-	if *flagVerbose {
-		logLevel.Set(slog.LevelDebug)
+	spec, err := NewDistributionSpec(*flagSpec)
+	if err != nil {
+		return err
+	}
+
+	val, err := spec.Query(*flagQuery)
+	if err != nil {
+		return err
+	}
+	// Using Println instead of logger since the results
+	// may be piped to another program.
+	fmt.Println(val)
+	return nil
+}
+
+func generateDistribution() error {
+	specPath := *flagSpec
+	if *flagSpec == "" {
+		return errNoSpecFlag
 	}
 
 	spec, err := NewDistributionSpec(specPath)

cmd/distrogen/testdata/generator/basic/golden/Makefile

@@ -1,3 +1,5 @@
+# NOTE: File generated by distrogen. Do not manually edit.
+
 # Variables filled in from distro specification
 VERSION ?= 0.121.0
 OTEL_VERSION ?= 0.121.0

cmd/distrogen/testdata/generator/custom_templates_subdir/golden/Makefile

@@ -1,3 +1,5 @@
+# NOTE: File generated by distrogen. Do not manually edit.
+
 # Variables filled in from distro specification
 VERSION ?= 0.121.0
 OTEL_VERSION ?= 0.121.0

cmd/distrogen/testdata/generator/no_docker_repo/golden/Makefile

@@ -1,3 +1,5 @@
+# NOTE: File generated by distrogen. Do not manually edit.
+
 # Variables filled in from distro specification
 VERSION ?= 0.121.0
 OTEL_VERSION ?= 0.121.0

components/google-built-opentelemetry-collector/Makefile

@@ -0,0 +1 @@
+include ../../make/component_dir.mk

components/google-built-opentelemetry-collector/exporter/googleservicecontrolexporter/Makefile

@@ -0,0 +1,3 @@
+include ../../../../make/maintenance.mk
+include ../../../../make/common.mk
+include ../../../../make/otel_component.mk

exporter/googleservicecontrolexporter/exporter_test.go → components/google-built-opentelemetry-collector/exporter/googleservicecontrolexporter/exporter_test.go

@@ -42,7 +42,7 @@ import (
 	"google.golang.org/protobuf/types/known/structpb"
 	"google.golang.org/protobuf/types/known/timestamppb"
 
-	"github.com/GoogleCloudPlatform/opentelemetry-operations-collector/exporter/googleservicecontrolexporter/internal/metadata"
+	"github.com/GoogleCloudPlatform/opentelemetry-operations-collector/components/google-built-opentelemetry-collector/exporter/googleservicecontrolexporter/internal/metadata"
 )
 
 const (

exporter/googleservicecontrolexporter/factory.go → components/google-built-opentelemetry-collector/exporter/googleservicecontrolexporter/factory.go

@@ -30,7 +30,7 @@ import (
 	"google.golang.org/grpc/credentials/insecure"
 	"google.golang.org/grpc/credentials/oauth"
 
-	"github.com/GoogleCloudPlatform/opentelemetry-operations-collector/exporter/googleservicecontrolexporter/internal/metadata"
+	"github.com/GoogleCloudPlatform/opentelemetry-operations-collector/components/google-built-opentelemetry-collector/exporter/googleservicecontrolexporter/internal/metadata"
 )
 
 var (

exporter/googleservicecontrolexporter/factory_test.go → components/google-built-opentelemetry-collector/exporter/googleservicecontrolexporter/factory_test.go

@@ -30,7 +30,7 @@ import (
 	"go.uber.org/zap"
 	"google.golang.org/grpc"
 
-	"github.com/GoogleCloudPlatform/opentelemetry-operations-collector/exporter/googleservicecontrolexporter/internal/metadata"
+	"github.com/GoogleCloudPlatform/opentelemetry-operations-collector/components/google-built-opentelemetry-collector/exporter/googleservicecontrolexporter/internal/metadata"
 )
 
 func TestCreateDefaultConfig(t *testing.T) {