diff --git a/.github/workflows/check-sync-module-dependencies.yml b/.github/workflows/check-sync-module-dependencies.yml new file mode 100644 index 00000000000..4a28629feed --- /dev/null +++ b/.github/workflows/check-sync-module-dependencies.yml @@ -0,0 +1,46 @@ +name: Check generate-module-dependencies +permissions: + contents: read +on: pull_request + +jobs: + check: + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: '1.25.x' + + - name: Run tests + run: | + cd tools/generate-module-dependencies + go test -v . + + - name: Run generate-module-dependencies + run: | + make generate-module-dependencies + + - name: Check for go.mod changes + run: | + # List changed go.mod files (added/modified/deleted) + CHANGED=$(git diff --name-only --ignore-all-space | grep -E '(^|/)go\.mod$' || true) + + if [ -n "$CHANGED" ]; then + echo "::error::go.mod files are out of sync with generate-module-dependencies." + echo "The following go.mod files changed:" + echo "$CHANGED" + echo + echo "Diff:" + git --no-pager diff -- $CHANGED || true + echo + echo "To fix locally:" + echo " Ensure that you update the dependency-replacements.yaml file and run make generate-module-dependencies" + exit 1 + fi \ No newline at end of file diff --git a/Makefile b/Makefile index ccb5e1a7b1b..1dbc72e13a5 100644 --- a/Makefile +++ b/Makefile @@ -49,6 +49,7 @@ ## generate-versioned-files Generate versioned files. ## generate-winmanifest Generate the Windows application manifest. ## generate-snmp Generate SNMP modules from prometheus/snmp_exporter for prometheus.exporter.snmp and bumps SNMP version in _index.md.t. +## generate-module-dependencies Generate replace directives from dependency-replacements.yaml and inject them into go.mod and builder-config.yaml. ## ## Other targets: ## @@ -257,6 +258,13 @@ else bash ./operations/helm/scripts/rebuild-tests.sh endif +generate-module-dependencies: +ifeq ($(USE_CONTAINER),1) + $(RERUN_IN_CONTAINER) +else + cd ./tools/generate-module-dependencies && $(GO_ENV) go generate +endif + generate-ui: ifeq ($(USE_CONTAINER),1) $(RERUN_IN_CONTAINER) diff --git a/dependency-replacements.yaml b/dependency-replacements.yaml new file mode 100644 index 00000000000..ee4710b180f --- /dev/null +++ b/dependency-replacements.yaml @@ -0,0 +1,131 @@ +# NOTE: replace directives below must always be *temporary*. +# +# Adding a replace directive to change a module to a fork of a module will +# only be accepted when a PR or an issue upstream has been opened to accept the new +# change. +# +# Contributors are expected to work with upstream to make their changes +# acceptable, and remove the `replace` directive as soon as possible. +# +# If upstream is unresponsive, you should consider making a hard fork +# (i.e., creating a new Go module with the same source) or picking a different +# dependency. + +# Modules define the target files where replaces will be applied. +# Each module specifies: +# - name: the name of the module +# - path: the path to the module's go.mod file +# - file_type: enum of what sort of generation pipeline will be used, currently the only available value is "mod" + +# Replace directives define the replacements that will be applied to the defined modules +# Each replace directive specifies: +# - comment: a brief explanation as to why the replace is necessary. This must contain a link to an upstream issue or a PR that is required to complete in order to remove the replace directive +# - dependency: the dependency to be replaced +# - replacement: the replacement for the dependency + +modules: + - name: alloy + path: go.mod + file_type: mod + +replaces: + - comment: > + TODO: remove this replace directive once the upstream issue is fixed: + https://github.com/prometheus/prometheus/issues/13842 + dependency: go.opentelemetry.io/collector/featuregate + replacement: github.com/grafana/opentelemetry-collector/featuregate v0.0.0-20240325174506-2fd1623b2ca0 + + - comment: Replace directives from Prometheus + dependency: github.com/fsnotify/fsnotify v1.8.0 + replacement: github.com/fsnotify/fsnotify v1.7.0 + + - comment: > + TODO: remove replace directive once there is a release of Prometheus + which addresses https://github.com/prometheus/prometheus/issues/14049, + for example, via this implementation: https://github.com/grafana/prometheus/pull/34 + dependency: github.com/prometheus/prometheus + replacement: github.com/grafana/prometheus v1.8.2-0.20251030104821-c9e0b31e9aeb + + - comment: Replace yaml.v2 with fork + dependency: gopkg.in/yaml.v2 + replacement: github.com/rfratto/go-yaml v0.0.0-20211119180816-77389c3526dc + + - comment: Replace directives from Loki — Azure SDK + dependency: github.com/Azure/azure-sdk-for-go + replacement: github.com/Azure/azure-sdk-for-go v68.0.0+incompatible + + - comment: Replace directives from Loki — Azure storage blob fork + dependency: github.com/Azure/azure-storage-blob-go + replacement: github.com/MasslessParticle/azure-storage-blob-go v0.14.1-0.20240322194317-344980fda573 + + - comment: Use fork of gocql that has gokit logs and Prometheus metrics + dependency: github.com/gocql/gocql + replacement: github.com/grafana/gocql v0.0.0-20200605141915-ba5dc39ece85 + + - comment: Insist on the optimised version of grafana/regexp + dependency: github.com/grafana/regexp + replacement: github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc + + - comment: Replace memberlist with Grafana fork which includes unmerged upstream fixes + dependency: github.com/hashicorp/memberlist + replacement: github.com/grafana/memberlist v0.3.1-0.20220714140823-09ffed8adbbe + + - comment: Use forked syslog implementation by leodido for continued support + dependency: github.com/influxdata/go-syslog/v3 + replacement: github.com/leodido/go-syslog/v4 v4.2.0 + + - comment: Replace thanos-io/objstore with Grafana fork + dependency: github.com/thanos-io/objstore + replacement: github.com/grafana/objstore v0.0.0-20250210100727-533688b5600d + + - comment: TODO - remove forks when changes are merged upstream — non-singleton cadvisor + dependency: github.com/google/cadvisor + replacement: github.com/grafana/cadvisor v0.0.0-20240729082359-1f04a91701e2 + + - comment: TODO - track exporter-package-v0.18.1 branch of grafana fork; remove once merged upstream + dependency: github.com/prometheus-community/postgres_exporter + replacement: github.com/grafana/postgres_exporter v0.0.0-20250930111128-c8f6a9f4d363 + + - comment: TODO - remove once PR is merged upstream - https://github.com/prometheus/mysqld_exporter/pull/774 + dependency: github.com/prometheus/mysqld_exporter + replacement: github.com/grafana/mysqld_exporter v0.17.2-0.20250226152553-be612e3fdedd + + - comment: > + TODO: replace node_exporter with custom fork for multi usage. + https://github.com/prometheus/node_exporter/pull/2812 + dependency: github.com/prometheus/node_exporter + replacement: github.com/grafana/node_exporter v0.18.1-grafana-r01.0.20251024135609-318b01780c89 + + - comment: Use Grafana fork of smimesign + dependency: github.com/github/smimesign + replacement: github.com/grafana/smimesign v0.2.1-0.20220408144937-2a5adf3481d3 + + - comment: Replace OpenTelemetry OBI with Grafana fork + dependency: go.opentelemetry.io/obi + replacement: github.com/grafana/opentelemetry-ebpf-instrumentation v1.3.13 + + - comment: Replace OpenTelemetry eBPF profiler with Grafana fork + dependency: go.opentelemetry.io/ebpf-profiler + replacement: github.com/grafana/opentelemetry-ebpf-profiler v0.0.202546-0.20251106085643-a00a0ef2a84c + + - comment: Update openshift/client-go to version compatible with structured-merge-diff v6 + dependency: github.com/openshift/client-go + replacement: github.com/openshift/client-go v0.0.0-20251015124057-db0dee36e235 + + - comment: Use Grafana's patched k8sattributesprocessor with support for k8s.io/client-go v0.34.1 + dependency: github.com/open-telemetry/opentelemetry-collector-contrib/processor/k8sattributesprocessor + replacement: github.com/grafana/opentelemetry-collector-contrib/processor/k8sattributesprocessor v0.0.0-20251021125353-73458b01ab23 + + - comment: > + Do not remove until bug in walqueue backwards compatibility is resolved: + https://github.com/deneonet/benc/issues/13 + dependency: github.com/deneonet/benc + replacement: github.com/deneonet/benc v1.1.7 + + - comment: Pin runc to v1.2.8 for compatibility with cadvisor requiring libcontainer/cgroups packages + dependency: github.com/opencontainers/runc + replacement: github.com/opencontainers/runc v1.2.8 + + - comment: Replace controller-runtime with pinned version + dependency: sigs.k8s.io/controller-runtime + replacement: sigs.k8s.io/controller-runtime v0.20.4 \ No newline at end of file diff --git a/dirwatch-test/.remote_config_hash b/dirwatch-test/.remote_config_hash new file mode 100644 index 00000000000..896f85ac8f1 --- /dev/null +++ b/dirwatch-test/.remote_config_hash @@ -0,0 +1 @@ +PR'ȢƿJ龏؊ďt)8 \ No newline at end of file diff --git a/dirwatch-test/config.yaml b/dirwatch-test/config.yaml new file mode 100644 index 00000000000..1f969bc9a63 --- /dev/null +++ b/dirwatch-test/config.yaml @@ -0,0 +1,32 @@ +service: + telemetry: + logs: + processors: + - batch: + exporter: + otlp: + endpoint: https://otlp-gateway-prod-us-central-0.grafana.net/otlp/v1/logs + headers: + - name: Authorization + value: Basic ${env:GCLOUD_BASIC_AUTH} + protocol: http/protobuf + metrics: + readers: + - periodic: + exporter: + otlp: + endpoint: https://otlp-gateway-prod-us-central-0.grafana.net/otlp/v1/metrics + headers: + - name: Authorization + value: Basic ${env:GCLOUD_BASIC_AUTH} + protocol: http/protobuf + traces: + processors: + - batch: + exporter: + otlp: + endpoint: https://otlp-gateway-prod-us-central-0.grafana.net/otlp/v1/traces + headers: + - name: Authorization + value: Basic ${env:GCLOUD_BASIC_AUTH} + protocol: http/protobuf diff --git a/go.mod b/go.mod index f8f3ab70e35..a0787643f28 100644 --- a/go.mod +++ b/go.mod @@ -2,6 +2,10 @@ module github.com/grafana/alloy go 1.25.5 +// This local replace is required for local development and testing of the syntax submodule. +// It is intentionally kept outside the generated block to avoid being overwritten by dependency management tools. +replace github.com/grafana/alloy/syntax => ./syntax + require ( cloud.google.com/go/pubsub v1.50.1 cloud.google.com/go/pubsub/v2 v2.0.0 @@ -116,7 +120,7 @@ require ( github.com/oklog/run v1.2.0 github.com/olekukonko/tablewriter v0.0.5 github.com/oliver006/redis_exporter v1.74.0 - github.com/open-telemetry/opentelemetry-collector-contrib/connector/countconnector v0.139.0 + github.com/open-telemetry/opentelemetry-collector-contrib/connector/countconnector v0.139.0 github.com/open-telemetry/opentelemetry-collector-contrib/connector/servicegraphconnector v0.139.0 github.com/open-telemetry/opentelemetry-collector-contrib/connector/spanmetricsconnector v0.139.0 github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awss3exporter v0.139.0 @@ -1006,94 +1010,81 @@ require ( golang.org/x/telemetry v0.0.0-20251008203120-078029d740a8 // indirect ) -// NOTE: replace directives below must always be *temporary*. -// -// Adding a replace directive to change a module to a fork of a module will -// only be accepted when a PR upstream has been opened to accept the new -// change. -// -// Contributors are expected to work with upstream to make their changes -// acceptable, and remove the `replace` directive as soon as possible. -// -// If upstream is unresponsive, you should consider making a hard fork -// (i.e., creating a new Go module with the same source) or picking a different -// dependency. +// Add exclude directives so Go doesn't pick old incompatible k8s.io/client-go +// versions. +exclude ( + k8s.io/client-go v8.0.0+incompatible + k8s.io/client-go v12.0.0+incompatible +) +// BEGIN GENERATED REPLACES - DO NOT EDIT MANUALLY // TODO: remove this replace directive once the upstream issue is fixed: https://github.com/prometheus/prometheus/issues/13842 -replace go.opentelemetry.io/collector/featuregate => github.com/grafana/opentelemetry-collector/featuregate v0.0.0-20240325174506-2fd1623b2ca0 // feature-gate-registration-error-handler branch +replace go.opentelemetry.io/collector/featuregate => github.com/grafana/opentelemetry-collector/featuregate v0.0.0-20240325174506-2fd1623b2ca0 // Replace directives from Prometheus replace github.com/fsnotify/fsnotify v1.8.0 => github.com/fsnotify/fsnotify v1.7.0 -// TODO: remove replace directive once: -// * There is a release of Prometheus which addresses https://github.com/prometheus/prometheus/issues/14049, -// for example, via this implementation: https://github.com/grafana/prometheus/pull/34 -replace github.com/prometheus/prometheus => github.com/grafana/prometheus v1.8.2-0.20251030104821-c9e0b31e9aeb // staleness_disabling_v3.7.3 branch +// TODO: remove replace directive once there is a release of Prometheus which addresses https://github.com/prometheus/prometheus/issues/14049, for example, via this implementation: https://github.com/grafana/prometheus/pull/34 +replace github.com/prometheus/prometheus => github.com/grafana/prometheus v1.8.2-0.20251030104821-c9e0b31e9aeb +// Replace yaml.v2 with fork replace gopkg.in/yaml.v2 => github.com/rfratto/go-yaml v0.0.0-20211119180816-77389c3526dc -// Replace directives from Loki -replace ( - github.com/Azure/azure-sdk-for-go => github.com/Azure/azure-sdk-for-go v68.0.0+incompatible - github.com/Azure/azure-storage-blob-go => github.com/MasslessParticle/azure-storage-blob-go v0.14.1-0.20240322194317-344980fda573 - // Use fork of gocql that has gokit logs and Prometheus metrics. - github.com/gocql/gocql => github.com/grafana/gocql v0.0.0-20200605141915-ba5dc39ece85 - // Insist on the optimised version of grafana/regexp - github.com/grafana/regexp => github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc - // Replace memberlist with our fork which includes some fixes that haven't been - // merged upstream yet. - github.com/hashicorp/memberlist => github.com/grafana/memberlist v0.3.1-0.20220714140823-09ffed8adbbe - // leodido fork his project to continue support - github.com/influxdata/go-syslog/v3 => github.com/leodido/go-syslog/v4 v4.2.0 - github.com/thanos-io/objstore => github.com/grafana/objstore v0.0.0-20250210100727-533688b5600d -) +// Replace directives from Loki — Azure SDK +replace github.com/Azure/azure-sdk-for-go => github.com/Azure/azure-sdk-for-go v68.0.0+incompatible -// TODO(rfratto): remove forks when changes are merged upstream -replace ( - // TODO(tpaschalis) this is to remove global instantiation of plugins - // and allow non-singleton components. - // https://github.com/grafana/cadvisor/tree/grafana-v0.47-noglobals - github.com/google/cadvisor => github.com/grafana/cadvisor v0.0.0-20240729082359-1f04a91701e2 +// Replace directives from Loki — Azure storage blob fork +replace github.com/Azure/azure-storage-blob-go => github.com/MasslessParticle/azure-storage-blob-go v0.14.1-0.20240322194317-344980fda573 - // TODO(dehaansa,cristiangreco): this tracks the exporter-package-v0.18.1 branch of the grafana fork, - // which we could get rid of once the changes are merged upstream. - github.com/prometheus-community/postgres_exporter => github.com/grafana/postgres_exporter v0.0.0-20250930111128-c8f6a9f4d363 +// Use fork of gocql that has gokit logs and Prometheus metrics +replace github.com/gocql/gocql => github.com/grafana/gocql v0.0.0-20200605141915-ba5dc39ece85 - // TODO(marctc): remove once this PR is merged upstream: https://github.com/prometheus/mysqld_exporter/pull/774 - github.com/prometheus/mysqld_exporter => github.com/grafana/mysqld_exporter v0.17.2-0.20250226152553-be612e3fdedd +// Insist on the optimised version of grafana/regexp +replace github.com/grafana/regexp => github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc - // TODO(marctc, mattdurham): Replace node_export with custom fork for multi usage. https://github.com/prometheus/node_exporter/pull/2812 - // this commit is on the refactor_collectors branch in the grafana fork. - github.com/prometheus/node_exporter => github.com/grafana/node_exporter v0.18.1-grafana-r01.0.20251024135609-318b01780c89 //refactor_collectors -) +// Replace memberlist with Grafana fork which includes unmerged upstream fixes +replace github.com/hashicorp/memberlist => github.com/grafana/memberlist v0.3.1-0.20220714140823-09ffed8adbbe -replace github.com/github/smimesign => github.com/grafana/smimesign v0.2.1-0.20220408144937-2a5adf3481d3 +// Use forked syslog implementation by leodido for continued support +replace github.com/influxdata/go-syslog/v3 => github.com/leodido/go-syslog/v4 v4.2.0 -// Submodules. -replace github.com/grafana/alloy/syntax => ./syntax +// Replace thanos-io/objstore with Grafana fork +replace github.com/thanos-io/objstore => github.com/grafana/objstore v0.0.0-20250210100727-533688b5600d -// Add exclude directives so Go doesn't pick old incompatible k8s.io/client-go -// versions. -exclude ( - k8s.io/client-go v8.0.0+incompatible - k8s.io/client-go v12.0.0+incompatible -) +// TODO - remove forks when changes are merged upstream — non-singleton cadvisor +replace github.com/google/cadvisor => github.com/grafana/cadvisor v0.0.0-20240729082359-1f04a91701e2 +// TODO - track exporter-package-v0.18.1 branch of grafana fork; remove once merged upstream +replace github.com/prometheus-community/postgres_exporter => github.com/grafana/postgres_exporter v0.0.0-20250930111128-c8f6a9f4d363 + +// TODO - remove once PR is merged upstream - https://github.com/prometheus/mysqld_exporter/pull/774 +replace github.com/prometheus/mysqld_exporter => github.com/grafana/mysqld_exporter v0.17.2-0.20250226152553-be612e3fdedd + +// TODO: replace node_exporter with custom fork for multi usage. https://github.com/prometheus/node_exporter/pull/2812 +replace github.com/prometheus/node_exporter => github.com/grafana/node_exporter v0.18.1-grafana-r01.0.20251024135609-318b01780c89 + +// Use Grafana fork of smimesign +replace github.com/github/smimesign => github.com/grafana/smimesign v0.2.1-0.20220408144937-2a5adf3481d3 + +// Replace OpenTelemetry OBI with Grafana fork replace go.opentelemetry.io/obi => github.com/grafana/opentelemetry-ebpf-instrumentation v1.3.13 +// Replace OpenTelemetry eBPF profiler with Grafana fork replace go.opentelemetry.io/ebpf-profiler => github.com/grafana/opentelemetry-ebpf-profiler v0.0.202546-0.20251106085643-a00a0ef2a84c -// Update openshift/client-go to a version compatible with structured-merge-diff v6 +// Update openshift/client-go to version compatible with structured-merge-diff v6 replace github.com/openshift/client-go => github.com/openshift/client-go v0.0.0-20251015124057-db0dee36e235 -// Use Grafana's patched k8sattributesprocessor that supports k8s.io/client-go v0.34.1 -// Adds RunWithContext and AddEventHandlerWithOptions methods to fake informers +// Use Grafana's patched k8sattributesprocessor with support for k8s.io/client-go v0.34.1 replace github.com/open-telemetry/opentelemetry-collector-contrib/processor/k8sattributesprocessor => github.com/grafana/opentelemetry-collector-contrib/processor/k8sattributesprocessor v0.0.0-20251021125353-73458b01ab23 -// Do not remove this until the bug breaking backwards compatibility is resolved & included in walqueue: https://github.com/deneonet/benc/issues/13 +// Do not remove until bug in walqueue backwards compatibility is resolved: https://github.com/deneonet/benc/issues/13 replace github.com/deneonet/benc => github.com/deneonet/benc v1.1.7 -// Pin runc to v1.1.12 to maintain compatibility with cadvisor that requires libcontainer/cgroups packages +// Pin runc to v1.2.8 for compatibility with cadvisor requiring libcontainer/cgroups packages replace github.com/opencontainers/runc => github.com/opencontainers/runc v1.2.8 +// Replace controller-runtime with pinned version replace sigs.k8s.io/controller-runtime => sigs.k8s.io/controller-runtime v0.20.4 + +// END GENERATED REPLACES diff --git a/go.sum b/go.sum index 43c6853d01d..345496a7285 100644 --- a/go.sum +++ b/go.sum @@ -1738,10 +1738,10 @@ github.com/onsi/ginkgo/v2 v2.23.4 h1:ktYTpKJAVZnDT4VjxSbiBenUjmlL/5QkBEocaWXiQus github.com/onsi/ginkgo/v2 v2.23.4/go.mod h1:Bt66ApGPBFzHyR+JO10Zbt0Gsp4uWxu5mIOTusL46e8= github.com/onsi/gomega v1.37.0 h1:CdEG8g0S133B4OswTDC/5XPSzE1OeP29QOioj2PID2Y= github.com/onsi/gomega v1.37.0/go.mod h1:8D9+Txp43QWKhM24yyOBEdpkzN8FvJyAwecBgsU4KU0= -github.com/open-telemetry/opentelemetry-collector-contrib/connector/countconnector v0.139.0 h1:L/mt81EynyqP8+Eo1RScotNMKodU8tzu5kdDVZsxzos= -github.com/open-telemetry/opentelemetry-collector-contrib/connector/countconnector v0.139.0/go.mod h1:aamgZ/EgklfjbFzEpZHJFw6558jhx7iVrH/BF4aIkDM= github.com/open-telemetry/opamp-go v0.22.0 h1:7UnsQgFFS7ffM09JQk+9aGVBAAlsLfcooZ9xvSYwxWM= github.com/open-telemetry/opamp-go v0.22.0/go.mod h1:339N71soCPrhHywbAcKUZJDODod581ZOxCpTkrl3zYQ= +github.com/open-telemetry/opentelemetry-collector-contrib/connector/countconnector v0.139.0 h1:L/mt81EynyqP8+Eo1RScotNMKodU8tzu5kdDVZsxzos= +github.com/open-telemetry/opentelemetry-collector-contrib/connector/countconnector v0.139.0/go.mod h1:aamgZ/EgklfjbFzEpZHJFw6558jhx7iVrH/BF4aIkDM= github.com/open-telemetry/opentelemetry-collector-contrib/connector/servicegraphconnector v0.139.0 h1:wsylHna5nuu6Jl+ow9uv3WJ0qKGJxg2aRORFUDa8SAs= github.com/open-telemetry/opentelemetry-collector-contrib/connector/servicegraphconnector v0.139.0/go.mod h1:LIR7KA0MIoHFQN/mO3zLEeK7piZL3cR6Ig10GfpS5UU= github.com/open-telemetry/opentelemetry-collector-contrib/connector/spanmetricsconnector v0.139.0 h1:USQU4VEL4Vi1rDm1am6LFjIvRGSOWhb+huw1OLIo3Eo= diff --git a/tools/generate-module-dependencies/README.md b/tools/generate-module-dependencies/README.md new file mode 100644 index 00000000000..abf0d855942 --- /dev/null +++ b/tools/generate-module-dependencies/README.md @@ -0,0 +1,46 @@ + +# generate-module-dependencies + +A small utility to keep Go module replace directives consistent across the repository from a single source of truth. + +## What it does + +- Reads dependency definitions from the project-level `dependency-replacements.yaml`. +- Renders the list of `replace` directives using a template. +- Injects the rendered block into target files (e.g., `go.mod`) between well-known markers. +- Runs `go mod tidy` for affected modules. + +Generated blocks are wrapped with: +``` +BEGIN GENERATED REPLACES - DO NOT EDIT MANUALLY ... END GENERATED REPLACES +``` + +**Do not edit anything between these markers**, manually—update `dependency-replacements.yaml` instead. Anything +changed manually within these markers will be overwritten during the next run. + +Please note that local replacement directives (ie pointing a dependency to a local module) are _not_ meant to be included +in `dependency-replacements.yaml`. These must be included separately in `go.mod` files, outside of the template boundaries + +## Usage + +- One-off run from repo root: + - `make generate-module-dependencies` + - or `cd tools/generate-module-dependencies && go generate` + +## Configuration + +All inputs come from `dependency-replacements.yaml`, which defines: +- Modules to update (name, path, file_type). +- Replace entries (dependency, replacement, optional comment). + +Comments are normalized (single-line) and included above the corresponding `replace` directive in generated output. + +## Troubleshooting + +- If a start marker exists without an end marker (or vice versa), generation fails—ensure both markers are present or absent together. +- If `go mod tidy` fails, fix the underlying module issues and rerun the command above. + +## Notes + +- This tool writes only the generated block; existing content outside the markers is preserved. +- The CI will run a check and fail if the output of the generate-module-dependencies step differs from the committed version. diff --git a/tools/generate-module-dependencies/cmd/sync-mod.go b/tools/generate-module-dependencies/cmd/sync-mod.go new file mode 100644 index 00000000000..9118e54e4a5 --- /dev/null +++ b/tools/generate-module-dependencies/cmd/sync-mod.go @@ -0,0 +1,52 @@ +package cmd + +import ( + "log" + "os" + + "github.com/grafana/replace-generator/internal" + "github.com/grafana/replace-generator/internal/helpers" + "github.com/spf13/cobra" +) + +var generateAndApplyReplaces = &cobra.Command{ + Use: "generate", + Short: "Generates replace directives as specified in the input dependency-replacements.yaml", + Run: func(cmd *cobra.Command, args []string) { + pathToYaml := cmd.Flag("dependency-yaml").Value.String() + pathToRoot := cmd.Flag("project-root").Value.String() + + fileHelper, err := helpers.NewFileHelper(pathToYaml, pathToRoot) + if err != nil { + log.Fatalf("Failed to create file helper: %v", err) + } + + projectReplaces, err := fileHelper.LoadProjectReplaces() + if err != nil { + log.Fatalf("Failed to load project replaces: %v", err) + } + + modByReplaceStr := internal.GenerateReplaces(fileHelper, projectReplaces) + internal.ApplyReplaces(fileHelper, projectReplaces, modByReplaceStr) + internal.TidyModules(fileHelper, projectReplaces) + }, +} + +func Execute() { + if err := NewRootCommand().Execute(); err != nil { + os.Exit(1) + } +} + +func NewRootCommand() *cobra.Command { + var rootCmd = &cobra.Command{ + Use: "sync-mod", + CompletionOptions: cobra.CompletionOptions{DisableDefaultCmd: true}, + } + + rootCmd.AddCommand(generateAndApplyReplaces) + generateAndApplyReplaces.Flags().String("dependency-yaml", "", "Relative path to the dependency-replacements.yaml file") + generateAndApplyReplaces.Flags().String("project-root", "", "Relative path to the project root") + + return rootCmd +} diff --git a/tools/generate-module-dependencies/e2e_test.go b/tools/generate-module-dependencies/e2e_test.go new file mode 100644 index 00000000000..6a543405c8e --- /dev/null +++ b/tools/generate-module-dependencies/e2e_test.go @@ -0,0 +1,79 @@ +package main + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/grafana/replace-generator/cmd" +) + +type testCase struct { + name string + testdataDir string +} + +var testCases = []testCase{ + { + name: "Basic", + testdataDir: "basic-mod", + }, + { + name: "UpdateExisting", + testdataDir: "update-existing-mod", + }, +} + +func TestE2E(t *testing.T) { + currentDir, err := os.Getwd() + if err != nil { + t.Fatalf("Failed to get current working directory: %v", err) + } + + command := cmd.NewRootCommand() + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + testdataDir := filepath.Join(currentDir, "testdata", tc.testdataDir) + goModPath := filepath.Join(testdataDir, "go.mod") + expectedPath := filepath.Join(testdataDir, "go.mod.expected") + dependencyYaml := filepath.Join("testdata", tc.testdataDir, "dependency-replacements-test.yaml") + projectRoot := filepath.Join("testdata", tc.testdataDir) + + originalGoMod, err := os.ReadFile(goModPath) + if err != nil { + t.Fatalf("Failed to read original go.mod: %v", err) + } + + // Restore the original go.mod after the test + defer func() { + if err := os.WriteFile(goModPath, originalGoMod, 0644); err != nil { + t.Errorf("Failed to restore original go.mod: %v", err) + } + }() + + command.SetArgs([]string{"generate", "--dependency-yaml", dependencyYaml, "--project-root", projectRoot}) + err = command.Execute() + if err != nil { + t.Fatalf("Failed to run command: %v", err) + } + + expectedContent, err := os.ReadFile(expectedPath) + if err != nil { + t.Fatalf("Failed to read expected go.mod: %v", err) + } + expectedGoMod := strings.TrimSpace(string(expectedContent)) + + actualContent, err := os.ReadFile(goModPath) + if err != nil { + t.Fatalf("Failed to read actual go.mod: %v", err) + } + actualGoMod := strings.TrimSpace(string(actualContent)) + + if actualGoMod != expectedGoMod { + t.Errorf("go.mod content mismatch.\nExpected:\n%s\n\nActual:\n%s", expectedGoMod, actualGoMod) + } + }) + } +} diff --git a/tools/generate-module-dependencies/generate.go b/tools/generate-module-dependencies/generate.go new file mode 100644 index 00000000000..8550a029ece --- /dev/null +++ b/tools/generate-module-dependencies/generate.go @@ -0,0 +1,3 @@ +package main + +//go:generate go run main.go generate --dependency-yaml=../../dependency-replacements.yaml --project-root=../.. diff --git a/tools/generate-module-dependencies/go.mod b/tools/generate-module-dependencies/go.mod new file mode 100644 index 00000000000..c5f42a9ea07 --- /dev/null +++ b/tools/generate-module-dependencies/go.mod @@ -0,0 +1,12 @@ +module github.com/grafana/replace-generator + +go 1.25.1 + +require gopkg.in/yaml.v3 v3.0.1 + +require github.com/spf13/cobra v1.10.1 + +require ( + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/spf13/pflag v1.0.10 // indirect +) diff --git a/tools/generate-module-dependencies/go.sum b/tools/generate-module-dependencies/go.sum new file mode 100644 index 00000000000..1c33d39098d --- /dev/null +++ b/tools/generate-module-dependencies/go.sum @@ -0,0 +1,13 @@ +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/cobra v1.10.1 h1:lJeBwCfmrnXthfAupyUTzJ/J4Nc1RsHC/mSRU2dll/s= +github.com/spf13/cobra v1.10.1/go.mod h1:7SmJGaTHFVBY0jW4NXGluQoLvhqFQM+6XSKD+P4XaB0= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= +github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/tools/generate-module-dependencies/internal/apply-replaces.go b/tools/generate-module-dependencies/internal/apply-replaces.go new file mode 100644 index 00000000000..dd5e0c71013 --- /dev/null +++ b/tools/generate-module-dependencies/internal/apply-replaces.go @@ -0,0 +1,96 @@ +package internal + +import ( + "fmt" + "log" + "os" + "strings" + + "github.com/grafana/replace-generator/internal/helpers" + "github.com/grafana/replace-generator/internal/types" +) + +func ApplyReplaces(fileHelper *helpers.FileHelper, projectReplaces *types.ProjectReplaces, modByReplaceStr map[string]*string) { + for _, module := range projectReplaces.Modules { + replacesStr := modByReplaceStr[module.Name] + if err := applyReplacesToModule(fileHelper, module, replacesStr); err != nil { + log.Fatalf("Failed to apply replaces to module %q: %v", module.Name, err) + } + log.Printf("Updated %s", module.Path) + } +} + +func applyReplacesToModule(dirs *helpers.FileHelper, module types.Module, replacesStr *string) error { + targetPath := dirs.ModuleTargetPath(module.Path) + + targetContent, err := os.ReadFile(targetPath) + if err != nil { + return fmt.Errorf("read target file %s: %w", targetPath, err) + } + + startMarker, endMarker, err := getMarkers(module.FileType) + if err != nil { + return fmt.Errorf("get markers for file type %q: %w", module.FileType, err) + } + + newContent, err := upsertGeneratedBlock(string(targetContent), *replacesStr, startMarker, endMarker) + if err != nil { + return fmt.Errorf("update generated block: %w", err) + } + + if err := os.WriteFile(targetPath, []byte(newContent), 0644); err != nil { + return fmt.Errorf("write target file %s: %w", targetPath, err) + } + + return nil +} + +func getCommentMarker(fileType types.FileType) (string, error) { + switch fileType { + case types.FileTypeMod: + return "//", nil + default: + return "", fmt.Errorf("unknown file_type %q", fileType) + } +} + +func getMarkers(fileType types.FileType) (startMarker, endMarker string, err error) { + commentSymbol, err := getCommentMarker(fileType) + if err != nil { + return "", "", err + } + + return fmt.Sprintf("%s BEGIN GENERATED REPLACES - DO NOT EDIT MANUALLY", commentSymbol), + fmt.Sprintf("%s END GENERATED REPLACES", commentSymbol), + nil +} + +// Upserts the generated block using the markers, or lack thereof, as a guide +func upsertGeneratedBlock(targetContent, replacement, startMarker, endMarker string) (string, error) { + startIdx := strings.Index(targetContent, startMarker) + startFound := startIdx != -1 + + if !startFound { + // No start marker: if the end marker exists anywhere, it's invalid. + if strings.Contains(targetContent, endMarker) { + return "", fmt.Errorf("found end marker without start marker") + } + + // Neither start not end marker found, append to the end of the file + targetContent = strings.TrimRight(targetContent, "\n") + return targetContent + "\n" + replacement, nil + } + + searchFrom := startIdx + len(startMarker) + endRel := strings.Index(targetContent[searchFrom:], endMarker) + if endRel == -1 { + // Start marker exists without an end marker, which is invalid + return "", fmt.Errorf("found start marker without end marker") + } + + endIdx := searchFrom + endRel + endOfMarker := endIdx + len(endMarker) + + // Replace [startIdx, endOfMarker) with replacement. + return targetContent[:startIdx] + replacement + targetContent[endOfMarker:], nil +} diff --git a/tools/generate-module-dependencies/internal/generate-replaces.go b/tools/generate-module-dependencies/internal/generate-replaces.go new file mode 100644 index 00000000000..1b634dddd12 --- /dev/null +++ b/tools/generate-module-dependencies/internal/generate-replaces.go @@ -0,0 +1,73 @@ +package internal + +import ( + "fmt" + "log" + "os" + "path/filepath" + "strings" + "text/template" + + "github.com/grafana/replace-generator/internal/helpers" + "github.com/grafana/replace-generator/internal/types" +) + +func GenerateReplaces(fileHelper *helpers.FileHelper, projectReplaces *types.ProjectReplaces) map[string]*string { + normalizeComments(projectReplaces.Replaces) + replaceTxtByMod := make(map[string]*string) + + for _, module := range projectReplaces.Modules { + str, err := generateReplacesForFileType(fileHelper, projectReplaces, module.FileType) + if err != nil { + log.Fatalf("Failed to generate replaces for module %q: %v", module.Name, err) + } + + replaceTxtByMod[module.Name] = str + } + + return replaceTxtByMod +} + +func normalizeComments(entries []types.ReplaceEntry) { + for i := range entries { + entries[i].Comment = strings.ReplaceAll(entries[i].Comment, "\n", " ") + entries[i].Comment = strings.TrimSpace(entries[i].Comment) + } +} + +func generateReplacesForFileType(dirs *helpers.FileHelper, projectReplaces *types.ProjectReplaces, fileType types.FileType) (*string, error) { + templatePath, err := dirs.TemplatePath(fileType) + + if err != nil { + return nil, fmt.Errorf("could not get template path: %w", err) + } + + str, err := generateFromTemplate(templatePath, projectReplaces.Replaces) + + if err != nil { + return nil, fmt.Errorf("could not execute template generation: %w", err) + } + + return str, nil +} + +func generateFromTemplate(templatePath string, entries []types.ReplaceEntry) (*string, error) { + tmplContent, err := os.ReadFile(templatePath) + if err != nil { + return nil, fmt.Errorf("could not read template %s: %w", templatePath, err) + } + + tmpl, err := template.New(filepath.Base(templatePath)).Parse(string(tmplContent)) + + if err != nil { + return nil, fmt.Errorf("could not parse template: %w", err) + } + + var buf strings.Builder + if err := tmpl.Execute(&buf, entries); err != nil { + return nil, fmt.Errorf("could not generate template: %w", err) + } + + str := buf.String() + return &str, nil +} diff --git a/tools/generate-module-dependencies/internal/helpers/files.go b/tools/generate-module-dependencies/internal/helpers/files.go new file mode 100644 index 00000000000..99f9ed20f97 --- /dev/null +++ b/tools/generate-module-dependencies/internal/helpers/files.go @@ -0,0 +1,86 @@ +package helpers + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/grafana/replace-generator/internal/types" + "gopkg.in/yaml.v3" +) + +type FileHelper struct { + // ScriptDir is the directory where the generate-module-dependencies code is located + ScriptDir string + + // ProjectRoot is the root directory of the Alloy project + ProjectRoot string + + // ProjectReplacesPath is the absolute path to dependency-replacements.yaml. + ProjectReplacesPath string +} + +func NewFileHelper(pathToDependencyReplacements string, projectRoot string) (*FileHelper, error) { + scriptDir, err := os.Getwd() + if err != nil { + return nil, fmt.Errorf("Failed to resolve working directory: %v", err) + } + + scriptDir, err = filepath.Abs(scriptDir) + if err != nil { + return nil, fmt.Errorf("Failed to resolve script directory: %v", err) + } + + absReplacesPath, err := filepath.Abs(pathToDependencyReplacements) + if err != nil { + return nil, fmt.Errorf("Failed to resolve %s: %v", pathToDependencyReplacements, err) + } + + return &FileHelper{ + ScriptDir: scriptDir, + ProjectRoot: projectRoot, + ProjectReplacesPath: absReplacesPath, + }, nil +} + +func (d *FileHelper) TemplatePath(fileType types.FileType) (string, error) { + var templateName string + var err error + + switch fileType { + case types.FileTypeMod: + templateName = "replaces-mod.tpl" + default: + err = fmt.Errorf("Unknown file_type %q (expected %q)", fileType, types.FileTypeMod) + } + return filepath.Join(d.ScriptDir, templateName), err +} + +func (d *FileHelper) ModuleTargetPath(modulePath string) string { + return filepath.Join(d.ProjectRoot, modulePath) +} + +func (d *FileHelper) ModuleDir(modulePath string) (string, error) { + moduleDir := filepath.Join(d.ProjectRoot, filepath.Dir(modulePath)) + abs, err := filepath.Abs(moduleDir) + + if err != nil { + return "", fmt.Errorf("Failed to resolve module directory %s: %v", moduleDir, err) + } + + return abs, nil +} + +func (d *FileHelper) LoadProjectReplaces() (*types.ProjectReplaces, error) { + data, err := os.ReadFile(d.ProjectReplacesPath) + if err != nil { + return nil, fmt.Errorf("could not read %s: %w", d.ProjectReplacesPath, err) + } + + var projectReplaces types.ProjectReplaces + if err := yaml.Unmarshal(data, &projectReplaces); err != nil { + return nil, fmt.Errorf("could not parse %s: %w", d.ProjectReplacesPath, err) + } + + return &projectReplaces, nil +} diff --git a/tools/generate-module-dependencies/internal/tidy-modules.go b/tools/generate-module-dependencies/internal/tidy-modules.go new file mode 100644 index 00000000000..e7290769c76 --- /dev/null +++ b/tools/generate-module-dependencies/internal/tidy-modules.go @@ -0,0 +1,36 @@ +package internal + +import ( + "fmt" + "log" + "os/exec" + + "github.com/grafana/replace-generator/internal/helpers" + "github.com/grafana/replace-generator/internal/types" +) + +func TidyModules(fileHelper *helpers.FileHelper, projectReplaces *types.ProjectReplaces) { + for _, module := range projectReplaces.Modules { + if err := runGoModTidy(fileHelper, module); err != nil { + log.Fatalf("Failed to run go mod tidy for module %q: %v", module.Name, err) + } + } +} + +func runGoModTidy(dirs *helpers.FileHelper, module types.Module) error { + moduleDir, err := dirs.ModuleDir(module.Path) + + if err != nil { + return err + } + + cmd := exec.Command("go", "mod", "tidy") + cmd.Dir = moduleDir + + log.Printf("Running go mod tidy in %s (module: %s)", moduleDir, module.Name) + if err := cmd.Run(); err != nil { + return fmt.Errorf("go mod tidy failed: %w", err) + } + + return nil +} diff --git a/tools/generate-module-dependencies/internal/types/types.go b/tools/generate-module-dependencies/internal/types/types.go new file mode 100644 index 00000000000..0a1fa9432e3 --- /dev/null +++ b/tools/generate-module-dependencies/internal/types/types.go @@ -0,0 +1,52 @@ +package types + +import ( + "fmt" + + "gopkg.in/yaml.v3" +) + +type FileType string + +const ( + FileTypeMod FileType = "mod" +) + +func (ft FileType) String() string { + return string(ft) +} + +func (ft *FileType) UnmarshalYAML(value *yaml.Node) error { + var s string + if err := value.Decode(&s); err != nil { + return err + } + + switch FileType(s) { + case FileTypeMod: + *ft = FileTypeMod + return nil + default: + return fmt.Errorf("invalid Module.file_type %q (expected %q)", s, FileTypeMod) + } +} + +// ReplaceEntry represents a single replace directive for a Go module dependency. +type ReplaceEntry struct { + Comment string `yaml:"comment"` + Dependency string `yaml:"dependency"` + Replacement string `yaml:"replacement"` +} + +// Module represents a Go module that needs replace directives applied. +type Module struct { + Name string `yaml:"name"` + Path string `yaml:"path"` + FileType FileType `yaml:"file_type"` +} + +// ProjectReplaces is the root structure of the dependency-replacements.yaml file. +type ProjectReplaces struct { + Modules []Module `yaml:"modules"` + Replaces []ReplaceEntry `yaml:"replaces"` +} diff --git a/tools/generate-module-dependencies/main.go b/tools/generate-module-dependencies/main.go new file mode 100644 index 00000000000..75aaee11d5f --- /dev/null +++ b/tools/generate-module-dependencies/main.go @@ -0,0 +1,7 @@ +package main + +import "github.com/grafana/replace-generator/cmd" + +func main() { + cmd.Execute() +} diff --git a/tools/generate-module-dependencies/replaces-mod.tpl b/tools/generate-module-dependencies/replaces-mod.tpl new file mode 100644 index 00000000000..0b2c8bfcf57 --- /dev/null +++ b/tools/generate-module-dependencies/replaces-mod.tpl @@ -0,0 +1,9 @@ +// BEGIN GENERATED REPLACES - DO NOT EDIT MANUALLY +{{- range . }} +{{- if .Comment }} +// {{ .Comment }} +{{- end }} +replace {{ .Dependency }} => {{ .Replacement }} +{{ end -}} +// END GENERATED REPLACES + diff --git a/tools/generate-module-dependencies/testdata/basic-mod/dependency-replacements-test.yaml b/tools/generate-module-dependencies/testdata/basic-mod/dependency-replacements-test.yaml new file mode 100644 index 00000000000..f4b5d4937dc --- /dev/null +++ b/tools/generate-module-dependencies/testdata/basic-mod/dependency-replacements-test.yaml @@ -0,0 +1,14 @@ +modules: + - name: test-module + path: go.mod + file_type: mod + +replaces: + - comment: Test replace for example.com/package + dependency: example.com/package + replacement: example.com/fork v1.0.0 + + - comment: Another test replace + dependency: github.com/test/dependency + replacement: github.com/test/fork v1.0.0 + diff --git a/tools/generate-module-dependencies/testdata/basic-mod/go.mod b/tools/generate-module-dependencies/testdata/basic-mod/go.mod new file mode 100644 index 00000000000..a642c0aa26b --- /dev/null +++ b/tools/generate-module-dependencies/testdata/basic-mod/go.mod @@ -0,0 +1,4 @@ +module test.example.com + +go 1.21 + diff --git a/tools/generate-module-dependencies/testdata/basic-mod/go.mod.expected b/tools/generate-module-dependencies/testdata/basic-mod/go.mod.expected new file mode 100644 index 00000000000..f13b7260caa --- /dev/null +++ b/tools/generate-module-dependencies/testdata/basic-mod/go.mod.expected @@ -0,0 +1,13 @@ +module test.example.com + +go 1.21 + +// BEGIN GENERATED REPLACES - DO NOT EDIT MANUALLY +// Test replace for example.com/package +replace example.com/package => example.com/fork v1.0.0 + +// Another test replace +replace github.com/test/dependency => github.com/test/fork v1.0.0 + +// END GENERATED REPLACES + diff --git a/tools/generate-module-dependencies/testdata/update-existing-mod/dependency-replacements-test.yaml b/tools/generate-module-dependencies/testdata/update-existing-mod/dependency-replacements-test.yaml new file mode 100644 index 00000000000..3e6330220b1 --- /dev/null +++ b/tools/generate-module-dependencies/testdata/update-existing-mod/dependency-replacements-test.yaml @@ -0,0 +1,10 @@ +modules: + - name: test-module + path: go.mod + file_type: mod + +replaces: + - comment: Updated replace + dependency: example.com/package + replacement: example.com/fork v1.1.0 + diff --git a/tools/generate-module-dependencies/testdata/update-existing-mod/go.mod b/tools/generate-module-dependencies/testdata/update-existing-mod/go.mod new file mode 100644 index 00000000000..8d6861788f7 --- /dev/null +++ b/tools/generate-module-dependencies/testdata/update-existing-mod/go.mod @@ -0,0 +1,10 @@ +module test.example.com + +go 1.21 + +// BEGIN GENERATED REPLACES - DO NOT EDIT MANUALLY +// Old comment +replace example.com/package => example.com/fork v1.0.0 + +// END GENERATED REPLACES + diff --git a/tools/generate-module-dependencies/testdata/update-existing-mod/go.mod.expected b/tools/generate-module-dependencies/testdata/update-existing-mod/go.mod.expected new file mode 100644 index 00000000000..bd0a31b1559 --- /dev/null +++ b/tools/generate-module-dependencies/testdata/update-existing-mod/go.mod.expected @@ -0,0 +1,10 @@ +module test.example.com + +go 1.21 + +// BEGIN GENERATED REPLACES - DO NOT EDIT MANUALLY +// Updated replace +replace example.com/package => example.com/fork v1.1.0 + +// END GENERATED REPLACES +