feat: Add unexposed otel engine and extension to codebase and change build structure

.github/workflows/check-generate-otel-collector-distro.yml

@@ -0,0 +1,37 @@
+name: Check generate-otel-collector-distro
+permissions:
+  contents: read
+on: pull_request
+
+jobs:
+  check:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+        with:
+          fetch-depth: 0
+          persist-credentials: false
+
+      - name: Set up Go
+        uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5
+        with:
+          go-version-file: go.mod
+          cache: false
+
+      - name: Run generate-otel-collector-distro
+        run: |
+          make generate-otel-collector-distro
+
+      - name: Check for generated file changes
+        run: |
+          # List changed files in the collector directory (excluding generator source files)
+          if [ -n "$(git status --porcelain -- collector/)" ]; then
+            echo "Error: generated collector files are out of sync with generate-otel-collector-distro."
+            echo "  Run: make generate-otel-collector-distro and commit the generated files"
+            git status -- collector/
+            git diff -- collector/
+            exit 1
+          fi
+

Makefile

@@ -79,19 +79,20 @@
 
 include tools/make/*.mk
 
-ALLOY_IMAGE          ?= grafana/alloy:latest
-ALLOY_IMAGE_WINDOWS  ?= grafana/alloy:windowsservercore-ltsc2022
-ALLOY_BINARY         ?= build/alloy
-SERVICE_BINARY       ?= build/alloy-service
-ALLOYLINT_BINARY     ?= build/alloylint
-JSONNET              ?= go run github.com/google/go-jsonnet/cmd/jsonnet@v0.20.0
-JB                   ?= go run github.com/jsonnet-bundler/jsonnet-bundler/cmd/jb@v0.6.0
-GOOS                 ?= $(shell go env GOOS)
-GOARCH               ?= $(shell go env GOARCH)
-GOARM                ?= $(shell go env GOARM)
-CGO_ENABLED          ?= 1
-RELEASE_BUILD        ?= 0
-GOEXPERIMENT         ?= $(shell go env GOEXPERIMENT)
+ALLOY_IMAGE          		?= grafana/alloy:latest
+ALLOY_IMAGE_WINDOWS  		?= grafana/alloy:windowsservercore-ltsc2022
+ALLOY_BINARY         		?= build/alloy
+SERVICE_BINARY       		?= build/alloy-service
+ALLOYLINT_BINARY     		?= build/alloylint
+BUILDER_VERSION      		?= v0.139.0
+JSONNET              		?= go run github.com/google/go-jsonnet/cmd/jsonnet@v0.20.0
+JB                   		?= go run github.com/jsonnet-bundler/jsonnet-bundler/cmd/jb@v0.6.0
+GOOS                 		?= $(shell go env GOOS)
+GOARCH               		?= $(shell go env GOARCH)
+GOARM                		?= $(shell go env GOARM)
+CGO_ENABLED          		?= 1
+RELEASE_BUILD        		?= 0
+GOEXPERIMENT         		?= $(shell go env GOEXPERIMENT)
 
 # Determine the golangci-lint binary path using Make functions where possible.
 # Priority: GOBIN, GOPATH/bin, PATH (via shell), Fallback Name.
@@ -195,11 +196,11 @@ integration-test-k8s: alloy-image
 .PHONY: binaries alloy
 binaries: alloy
 
-alloy:
+alloy: generate-otel-collector-distro
 ifeq ($(USE_CONTAINER),1)
 	$(RERUN_IN_CONTAINER)
 else
-	$(GO_ENV) go build $(GO_FLAGS) -o $(ALLOY_BINARY) .
+	cd ./collector && $(GO_ENV) go build $(GO_FLAGS) -o ../$(ALLOY_BINARY) .
 endif
 
 # alloy-service is not included in binaries since it's Windows-only.
@@ -243,8 +244,8 @@ alloy-image-windows:
 # Targets for generating assets
 #
 
-.PHONY: generate generate-helm-docs generate-helm-tests generate-ui generate-winmanifest generate-snmp generate-rendered-mixin
-generate: generate-helm-docs generate-helm-tests generate-ui generate-docs generate-winmanifest generate-snmp generate-rendered-mixin
+.PHONY: generate generate-helm-docs generate-helm-tests generate-ui generate-winmanifest generate-snmp generate-rendered-mixin generate-module-dependencies generate-otel-collector-distro
+generate: generate-helm-docs generate-helm-tests generate-ui generate-docs generate-winmanifest generate-snmp generate-rendered-mixin generate-module-dependencies generate-otel-collector-distro
 
 generate-helm-docs:
 ifeq ($(USE_CONTAINER),1)
@@ -267,6 +268,14 @@ else
 	cd ./tools/generate-module-dependencies && $(GO_ENV) go generate
 endif
 
+generate-otel-collector-distro:
+ifeq ($(USE_CONTAINER),1)
+	$(RERUN_IN_CONTAINER)
+else
+	# Here we clear the GOOS and GOARCH env variables so we're not accidentally cross compiling the builder tool within generate
+	cd ./collector && GOOS= GOARCH= BUILDER_VERSION=$(BUILDER_VERSION) go generate
+endif
+
 generate-ui:
 ifeq ($(USE_CONTAINER),1)
 	$(RERUN_IN_CONTAINER)

collector/README.md

@@ -0,0 +1,74 @@
+# Alloy Collector Distro
+
+This directory contains the build configuration and generated code for the **Alloy Collector Distro**, an OpenTelemetry Collector distribution that embeds Alloy's OTel components and integrates with the Alloy CLI. This also means that it contains the generated build root of the Alloy project.
+
+## Overview
+
+The Alloy Collector Distro is built using the [OpenTelemetry Collector Builder (OCB)](https://github.com/open-telemetry/opentelemetry-collector-builder) tool. Components are defined in an OCB configuration file, and the build process generates the necessary Go code, including the `main.go` file and `go.mod` dependencies.
+
+## Directory Structure
+
+```
+collector/
+├── builder-config.yaml        # OCB configuration file (source)
+├── generate.go                # Entry point for code generation (source)
+├── generator/                 # Custom templating and generation code (source)
+│   ├── generator.go          # Generator tool that post-processes OCB output
+│   └── main_alloy.tpl        # Template for main_alloy.go
+│
+├── main.go                    # Generated by OCB (DO NOT EDIT)
+├── main_alloy.go              # Generated by generator (DO NOT EDIT)
+├── main_others.go             # Generated by OCB (DO NOT EDIT)
+├── main_windows.go            # Generated by OCB (DO NOT EDIT)
+├── components.go              # Generated by OCB (DO NOT EDIT)
+├── go.mod                     # Generated by OCB (DO NOT EDIT)
+└── go.sum                     # Generated by OCB (DO NOT EDIT)
+```
+
+## Build Process
+
+The build process consists of three main steps:
+
+### 1. OCB Generation (`go run go.opentelemetry.io/collector/cmd/builder@${BUILDER_VERSION}`)
+
+The BUILDER_VERSION is defined in the root Makefile, and should be kept in sync with our general OTel component versioning
+
+The OCB tool reads `builder-config.yaml` and generates:
+- `main.go` - The main entry point for the collector
+- `components.go` - Component factories and registrations
+- `go.mod` and `go.sum` - Module dependencies
+- Other platform-specific main files (`main_others.go`, `main_windows.go`)
+
+The `builder-config.yaml` file defines:
+- General collector distribution metadata
+- The Open Telemetry components to include in the distribution (receivers/processors/exporters etc)
+- Replace directives, which are automated/generated to keep in sync with `dependency-replacements.yaml` in the root of the project
+
+### 2. Go Module Tidying (`go mod tidy`)
+
+Ensures the generated `go.mod` is properly formatted and all dependencies are resolved.
+
+### 3. Custom Generator (`go run ./generator/generator.go`)
+
+The custom generator performs two tasks:
+
+1. **Generates `main_alloy.go`** - Creates a wrapper that integrates the OTel collector command into Alloy's CLI as a subcommand (`alloy otel`)
+2. **Modifies `main.go`** - Replaces `otelcol.NewCommand(params)` with `newAlloyCommand(params)` to use the Alloy-integrated command
+
+## Building Locally
+
+When building locally via `make alloy`, the entire chain of replacement and collector generation runs automatically. This ensures a smooth development experience where you don't need to manually run generation commands before building.
+
+To manually trigger generation:
+
+```bash
+make generate-otel-collector-distro
+```
+
+## CI Considerations
+
+If your work involves anything that would modify the generated files, please commit them with your changes before pushing to github. We have a github workflow that will check that the output of the collector generation matches what has been checked in, and this will fail if you do not include changes to these files. 
+
+## A Note On Replace Directives
+
+The `replaces` section in `builder-config.yaml` must be kept in sync with the replace directives in the root `go.mod` file as well as the `go.mod` file in the alloy engine extension. The Alloy project uses the `dependency-replacements.yaml` file found in the root of the project to generate replace directives. If you modify a replacement, please run `make generate-module-dependencies` to sync things up. There is also a github workflow that will check against this. 

collector/builder-config.yaml

@@ -0,0 +1,95 @@
+dist:
+  module: github.com/grafana/alloy/otel_engine
+  name: alloy
+  description: Alloy OTel Collector distribution.
+  version: v1.12.1 # x-release-please-version
+  output_path: .
+
+extensions:
+  - gomod: go.opentelemetry.io/collector/extension/zpagesextension v0.139.0
+  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/extension/healthcheckextension v0.139.0
+  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/extension/pprofextension v0.139.0
+  - gomod: github.com/grafana/alloy/extension/alloyengine v0.1.0
+
+exporters:
+  - gomod: go.opentelemetry.io/collector/exporter/debugexporter v0.139.0
+  - gomod: go.opentelemetry.io/collector/exporter/nopexporter v0.139.0
+  - gomod: go.opentelemetry.io/collector/exporter/otlpexporter v0.139.0
+  - gomod: go.opentelemetry.io/collector/exporter/otlphttpexporter v0.139.0
+  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/exporter/fileexporter v0.139.0
+  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/exporter/kafkaexporter v0.139.0
+  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/exporter/prometheusexporter v0.139.0
+  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/exporter/prometheusremotewriteexporter v0.139.0
+  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/exporter/zipkinexporter v0.139.0
+
+processors:
+  - gomod: go.opentelemetry.io/collector/processor/batchprocessor v0.139.0
+  - gomod: go.opentelemetry.io/collector/processor/memorylimiterprocessor v0.139.0
+  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/attributesprocessor v0.139.0
+  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourceprocessor v0.139.0
+  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/spanprocessor v0.139.0
+  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/probabilisticsamplerprocessor v0.139.0
+  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/filterprocessor v0.139.0
+
+receivers:
+  - gomod: go.opentelemetry.io/collector/receiver/otlpreceiver v0.139.0
+
+connectors:
+  - gomod: go.opentelemetry.io/collector/connector/forwardconnector v0.139.0
+
+providers:
+  - gomod: go.opentelemetry.io/collector/confmap/provider/envprovider v1.45.0
+  - gomod: go.opentelemetry.io/collector/confmap/provider/fileprovider v1.45.0
+  - gomod: go.opentelemetry.io/collector/confmap/provider/httpprovider v1.45.0
+  - gomod: go.opentelemetry.io/collector/confmap/provider/httpsprovider v1.45.0
+  - gomod: go.opentelemetry.io/collector/confmap/provider/yamlprovider v1.45.0
+
+replaces:
+  # Local modules.
+  - github.com/grafana/alloy => ../
+  - github.com/grafana/alloy/syntax => ../syntax
+  - github.com/grafana/alloy/extension/alloyengine => ../extension/alloyengine
+  # BEGIN GENERATED REPLACES - DO NOT EDIT MANUALLY
+  # TODO: remove this replace directive once the upstream issue is fixed: https://github.com/prometheus/prometheus/issues/13842
+  - go.opentelemetry.io/collector/featuregate => github.com/grafana/opentelemetry-collector/featuregate v0.0.0-20240325174506-2fd1623b2ca0
+  # Replace yaml.v2 with fork
+  - gopkg.in/yaml.v2 => github.com/rfratto/go-yaml v0.0.0-20211119180816-77389c3526dc
+  # Replace directives from Loki — Azure SDK
+  - github.com/Azure/azure-sdk-for-go => github.com/Azure/azure-sdk-for-go v68.0.0+incompatible
+  # Replace directives from Loki — Azure storage blob fork
+  - github.com/Azure/azure-storage-blob-go => github.com/MasslessParticle/azure-storage-blob-go v0.14.1-0.20240322194317-344980fda573
+  # Use fork of gocql that has gokit logs and Prometheus metrics
+  - github.com/gocql/gocql => github.com/grafana/gocql v0.0.0-20200605141915-ba5dc39ece85
+  # Insist on the optimised version of grafana/regexp
+  - github.com/grafana/regexp => github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc
+  # Replace memberlist with Grafana fork which includes unmerged upstream fixes
+  - github.com/hashicorp/memberlist => github.com/grafana/memberlist v0.3.1-0.20220714140823-09ffed8adbbe
+  # Use forked syslog implementation by leodido for continued support
+  - github.com/influxdata/go-syslog/v3 => github.com/leodido/go-syslog/v4 v4.3.0
+  # Replace thanos-io/objstore with Grafana fork
+  - github.com/thanos-io/objstore => github.com/grafana/objstore v0.0.0-20250210100727-533688b5600d
+  # TODO - remove forks when changes are merged upstream — non-singleton cadvisor
+  - github.com/google/cadvisor => github.com/grafana/cadvisor v0.0.0-20240729082359-1f04a91701e2
+  # TODO - track exporter-package-v0.18.1 branch of grafana fork; remove once merged upstream
+  - github.com/prometheus-community/postgres_exporter => github.com/grafana/postgres_exporter v0.0.0-20250930111128-c8f6a9f4d363
+  # TODO - remove once PR is merged upstream - https://github.com/prometheus/mysqld_exporter/pull/774
+  - github.com/prometheus/mysqld_exporter => github.com/grafana/mysqld_exporter v0.17.2-0.20250226152553-be612e3fdedd
+  # TODO: replace node_exporter with custom fork for multi usage. https://github.com/prometheus/node_exporter/pull/2812
+  - github.com/prometheus/node_exporter => github.com/grafana/node_exporter v0.18.1-grafana-r01.0.20251024135609-318b01780c89
+  # Use Grafana fork of smimesign
+  - github.com/github/smimesign => github.com/grafana/smimesign v0.2.1-0.20220408144937-2a5adf3481d3
+  # Replace OpenTelemetry OBI with Grafana fork
+  - go.opentelemetry.io/obi => github.com/grafana/opentelemetry-ebpf-instrumentation v1.3.13
+  # Replace OpenTelemetry eBPF profiler with Grafana fork
+  - go.opentelemetry.io/ebpf-profiler => github.com/grafana/opentelemetry-ebpf-profiler v0.0.202550-0.20251216083346-f910fc4db23e
+  # Update openshift/client-go to version compatible with structured-merge-diff v6
+  - github.com/openshift/client-go => github.com/openshift/client-go v0.0.0-20251015124057-db0dee36e235
+  # Do not remove until bug in walqueue backwards compatibility is resolved: https://github.com/deneonet/benc/issues/13
+  - github.com/deneonet/benc => github.com/deneonet/benc v1.1.7
+  # Pin runc to v1.2.8 for compatibility with cadvisor requiring libcontainer/cgroups packages
+  - github.com/opencontainers/runc => github.com/opencontainers/runc v1.2.8
+  # Replace controller-runtime with pinned version
+  - sigs.k8s.io/controller-runtime => sigs.k8s.io/controller-runtime v0.20.4
+  # Point to fork that uses prometheus v0.308.0, changes here are mostly related to type name updates. This can be removed once the upstream dependency points to prometheus v0.308.0 and above
+  - github.com/open-telemetry/opentelemetry-collector-contrib/exporter/prometheusremotewriteexporter => github.com/grafana/opentelemetry-collector-contrib/exporter/prometheusremotewriteexporter v0.0.0-20260108134526-9817e6c361c9
+  # END GENERATED REPLACES