From 72adc4beb8740ca4bd7fcc139a8e90803a927f5b Mon Sep 17 00:00:00 2001 From: Tyler Yahn Date: Wed, 18 Feb 2026 14:20:07 -0800 Subject: [PATCH 1/3] PoC: OBI integration with collector-contrib dist --- .gitignore | 1 + .gitmodules | 3 + Makefile | 28 ++- OBI_RECEIVER.md | 244 ++++++++++++++++++++ distributions/otelcol-contrib/manifest.yaml | 8 + internal/obi | 1 + 6 files changed, 283 insertions(+), 2 deletions(-) create mode 100644 .gitmodules create mode 100644 OBI_RECEIVER.md create mode 160000 internal/obi diff --git a/.gitignore b/.gitignore index 9d76ce30e..ece9c1cdc 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ dist/ .tools .DS_Store .zed +distributions/*/windows-installer.wxs diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 000000000..488f57172 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "internal/obi"] + path = internal/obi + url = https://github.com/open-telemetry/opentelemetry-ebpf-instrumentation.git diff --git a/Makefile b/Makefile index 90f2388fa..e247b420d 100644 --- a/Makefile +++ b/Makefile @@ -25,7 +25,7 @@ BINARIES ?= "builder,opampsupervisor" ci: check build check: ensure-goreleaser-up-to-date validate-components validate-version-consistency -build: go ocb +build: go ocb check-obi-generated @./scripts/build.sh -d "${DISTRIBUTIONS}" -b ${OTELCOL_BUILDER} generate: generate-sources generate-goreleaser @@ -33,9 +33,33 @@ generate: generate-sources generate-goreleaser generate-goreleaser: go @./scripts/generate-goreleaser.sh -d "${DISTRIBUTIONS}" -b "${BINARIES}" -g ${GO} -generate-sources: go ocb generate-msi +generate-sources: go ocb generate-msi check-obi-generated @./scripts/build.sh -d "${DISTRIBUTIONS}" -s true -b ${OTELCOL_BUILDER} +# Generate OBI eBPF artifacts (requires Docker) +.PHONY: generate-obi +generate-obi: + @if [ ! -d "internal/obi" ]; then \ + echo "Error: OBI submodule not initialized."; \ + echo "Run: git submodule update --init --recursive"; \ + exit 1; \ + fi + @echo "Generating OBI eBPF artifacts (this takes 2-5 minutes)..." + @cd internal/obi && $(MAKE) docker-generate + @echo "✓ OBI eBPF artifacts generated successfully" + +# Check if OBI eBPF artifacts have been generated +.PHONY: check-obi-generated +check-obi-generated: + @if [ -d "internal/obi" ]; then \ + if ! find internal/obi -name "*_bpfel.go" | grep -q .; then \ + echo "Error: OBI eBPF artifacts not generated."; \ + echo "Run: make generate-obi"; \ + echo "This requires Docker to be available."; \ + exit 1; \ + fi \ + fi + generate-msi: go ocb $(GO) run cmd/msi-generator/main.go -d "${DISTRIBUTIONS}" diff --git a/OBI_RECEIVER.md b/OBI_RECEIVER.md new file mode 100644 index 000000000..2ccc27e83 --- /dev/null +++ b/OBI_RECEIVER.md @@ -0,0 +1,244 @@ +# OBI Receiver Integration + +This document explains how OBI (eBPF-based zero-code instrumentation) is integrated into the OpenTelemetry Collector distributions. + +## Overview + +The OBI receiver is included in the `otelcol-contrib` distribution as an external component. It is maintained in its own repository at [opentelemetry-ebpf-instrumentation](https://github.com/open-telemetry/opentelemetry-ebpf-instrumentation) and included here as a git submodule. + +## Architecture + +### Why a Submodule? + +OBI requires eBPF code generation (converting C code to Go) before the module can be built. The generated files are: +- **Not committed** to OBI's git repository (policy decision) +- **Platform-specific** (Linux kernel versions) +- **Large** (~10MB of binary objects and generated Go code) + +Therefore, we need: +1. The OBI source code (git submodule at `internal/obi`) +2. A generation step before building (`make generate-obi`) +3. A replace directive to point the builder to the local submodule + +### Integration Pattern + +The `otelcol-contrib` distribution manifest includes: + +```yaml +receivers: + - gomod: go.opentelemetry.io/obi v0.5.0 + import: go.opentelemetry.io/obi/collector + +replaces: + - go.opentelemetry.io/obi => ../../internal/obi +``` + +This uses the OpenTelemetry Collector Builder's `import:` field to directly reference the OBI collector package without needing a wrapper module. + +## Building with OBI + +### Prerequisites + +- Git with submodule support +- Docker (for eBPF code generation) +- Standard Go build tools + +### First Time Setup + +```bash +# Clone with submodules +git clone --recurse-submodules https://github.com/open-telemetry/opentelemetry-collector-releases.git + +# Or initialize submodules if already cloned +git submodule update --init --recursive + +# Generate OBI eBPF artifacts (requires Docker, takes 2-5 minutes) +make generate-obi +``` + +### Building Distributions + +Once OBI artifacts are generated: + +```bash +# Build all distributions +make build + +# Or build specific distribution +make build DISTRIBUTIONS="otelcol-contrib" +``` + +The build process will automatically check that OBI artifacts exist before building. + +## CI/CD Integration + +### Required Workflow Changes + +All CI workflows that build distributions must: + +1. **Initialize submodules:** + ```yaml + - uses: actions/checkout@v6 + with: + submodules: 'recursive' + ``` + +2. **Generate OBI artifacts (Linux only):** + ```yaml + - name: Generate OBI eBPF artifacts + if: runner.os == 'Linux' + run: make generate-obi + ``` + +3. **Build as normal:** + ```yaml + - name: Build distributions + run: make build + ``` + +### Build Time Impact + +- **Without caching:** Adds ~2-5 minutes to Linux builds +- **With caching:** Near-zero impact after first build +- **Submodule checkout:** Adds ~10-20 seconds + +### Artifact Caching + +To optimize CI builds, cache generated artifacts: + +```yaml +- name: Cache OBI artifacts + id: cache-obi + uses: actions/cache@v3 + with: + path: | + internal/obi/**/*_bpfel.go + internal/obi/**/*_bpfeb.go + internal/obi/**/*.o + key: obi-artifacts-${{ hashFiles('internal/obi/.git/HEAD') }} + +- name: Generate OBI eBPF artifacts + if: runner.os == 'Linux' && steps.cache-obi.outputs.cache-hit != 'true' + run: make generate-obi +``` + +## Maintenance + +### Updating OBI Version + +1. **Update the submodule to a new commit:** + ```bash + cd internal/obi + git fetch + git checkout v0.6.0 # or specific commit + cd ../.. + git add internal/obi + ``` + +2. **Update the manifest version:** + ```bash + # Edit distributions/otelcol-contrib/manifest.yaml + # Change: gomod: go.opentelemetry.io/obi v0.5.0 + # To: gomod: go.opentelemetry.io/obi v0.6.0 + ``` + +3. **Regenerate artifacts:** + ```bash + make generate-obi + ``` + +4. **Test the build:** + ```bash + make build DISTRIBUTIONS="otelcol-contrib" + ``` + +5. **Commit the changes:** + ```bash + git commit -m "Update OBI receiver to v0.6.0" + ``` + +### Troubleshooting + +#### Error: "OBI submodule not initialized" + +**Solution:** +```bash +git submodule update --init --recursive +``` + +#### Error: "OBI eBPF artifacts not generated" + +**Solution:** +```bash +make generate-obi +``` + +If generation fails, check: +- Docker is installed and running +- Docker can pull images +- You have sufficient disk space (generation needs ~500MB temp space) + +#### Slow Builds + +**Solution:** Implement CI artifact caching (see above) + +## Platform Support + +- ✅ **Linux (x86_64, arm64):** Full support with eBPF instrumentation +- ⚠️ **Other platforms:** Receiver included but runs as no-op (OBI uses build tags) + +The OBI code itself handles platform detection: +- `collector/factory_linux.go` - Full implementation +- `collector/factory_others.go` - No-op stub + +## Why Not in Collector-Contrib? + +The OBI receiver is an **external component** maintained in its own repository, not a **donated component**. According to collector-contrib documentation: + +> "you can just host it in your own repository as a Go module" + +The collector-releases repository is the appropriate place for: +- Packaging external components into distributions +- Managing build-time dependencies (like submodules) +- Distribution-specific configurations + +The collector-contrib repository is for: +- Components donated to the project +- Code maintained by the OpenTelemetry community +- Components that follow contrib's lifecycle + +## Future Improvements + +### Option 1: Pre-Generated Artifacts (Recommended) + +If OBI publishes pre-generated artifacts as GitHub release assets: + +1. Remove the submodule +2. Add a download step: `scripts/download-obi-artifacts.sh` +3. Remove the replace directive (use published module) + +This would eliminate all generation complexity. + +### Option 2: OBI Commits Artifacts + +If OBI changes policy and commits generated files to git: + +1. Remove the submodule +2. Remove generation targets from Makefile +3. Remove the replace directive +4. Reference OBI as a normal Go dependency + +This would be the simplest solution but requires OBI team approval. + +## References + +- **OBI Repository:** https://github.com/open-telemetry/opentelemetry-ebpf-instrumentation +- **OBI Collector Example:** https://github.com/open-telemetry/opentelemetry-ebpf-instrumentation/tree/main/examples/otel-collector +- **Collector Builder Docs:** https://opentelemetry.io/docs/collector/custom-collector/ + +## Questions? + +For questions about: +- **OBI functionality:** Open an issue in the OBI repository +- **Build integration:** Open an issue in this repository +- **Distribution inclusion:** Discuss in Collector SIG meetings diff --git a/distributions/otelcol-contrib/manifest.yaml b/distributions/otelcol-contrib/manifest.yaml index e40d1cf9e..dbfecbdd9 100644 --- a/distributions/otelcol-contrib/manifest.yaml +++ b/distributions/otelcol-contrib/manifest.yaml @@ -131,6 +131,9 @@ processors: receivers: - gomod: go.opentelemetry.io/collector/receiver/nopreceiver v0.147.0 - gomod: go.opentelemetry.io/collector/receiver/otlpreceiver v0.147.0 + # OBI (eBPF Instrumentation) - requires submodule at internal/obi + - gomod: go.opentelemetry.io/obi v0.5.0 + import: go.opentelemetry.io/obi/collector - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/activedirectorydsreceiver v0.147.0 - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/aerospikereceiver v0.147.0 - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/apachereceiver v0.147.0 @@ -259,3 +262,8 @@ providers: - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/confmap/provider/s3provider v0.147.0 - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/confmap/provider/secretsmanagerprovider v0.147.0 - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/confmap/provider/googlesecretmanagerprovider v0.147.0 + +replaces: + # OBI submodule - required for eBPF artifact generation + # Path is relative to distributions/otelcol-contrib/ + - go.opentelemetry.io/obi => ../../../internal/obi diff --git a/internal/obi b/internal/obi new file mode 160000 index 000000000..754e7882f --- /dev/null +++ b/internal/obi @@ -0,0 +1 @@ +Subproject commit 754e7882f2d48302863570d3a64d0306352f1426 From 5bce5bdcaa7f013df03ccb99fab8636f7de79871 Mon Sep 17 00:00:00 2001 From: Tyler Yahn Date: Mon, 23 Feb 2026 17:11:55 -0800 Subject: [PATCH 2/3] Replace submodule integration with tagged source prep --- .gitignore | 2 + .gitmodules | 3 - Makefile | 30 +-- OBI_RECEIVER.md | 218 +++----------------- distributions/otelcol-contrib/manifest.yaml | 7 +- internal/obi | 1 - scripts/prepare-obi.sh | 76 +++++++ 7 files changed, 114 insertions(+), 223 deletions(-) delete mode 100644 .gitmodules delete mode 160000 internal/obi create mode 100755 scripts/prepare-obi.sh diff --git a/.gitignore b/.gitignore index ece9c1cdc..d73d6fb61 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,5 @@ dist/ .DS_Store .zed distributions/*/windows-installer.wxs +internal/obi/ +internal/obi-src/ diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index 488f57172..000000000 --- a/.gitmodules +++ /dev/null @@ -1,3 +0,0 @@ -[submodule "internal/obi"] - path = internal/obi - url = https://github.com/open-telemetry/opentelemetry-ebpf-instrumentation.git diff --git a/Makefile b/Makefile index e247b420d..aca1a3b28 100644 --- a/Makefile +++ b/Makefile @@ -25,7 +25,7 @@ BINARIES ?= "builder,opampsupervisor" ci: check build check: ensure-goreleaser-up-to-date validate-components validate-version-consistency -build: go ocb check-obi-generated +build: go ocb prepare-obi @./scripts/build.sh -d "${DISTRIBUTIONS}" -b ${OTELCOL_BUILDER} generate: generate-sources generate-goreleaser @@ -33,32 +33,12 @@ generate: generate-sources generate-goreleaser generate-goreleaser: go @./scripts/generate-goreleaser.sh -d "${DISTRIBUTIONS}" -b "${BINARIES}" -g ${GO} -generate-sources: go ocb generate-msi check-obi-generated +generate-sources: go ocb generate-msi prepare-obi @./scripts/build.sh -d "${DISTRIBUTIONS}" -s true -b ${OTELCOL_BUILDER} -# Generate OBI eBPF artifacts (requires Docker) -.PHONY: generate-obi -generate-obi: - @if [ ! -d "internal/obi" ]; then \ - echo "Error: OBI submodule not initialized."; \ - echo "Run: git submodule update --init --recursive"; \ - exit 1; \ - fi - @echo "Generating OBI eBPF artifacts (this takes 2-5 minutes)..." - @cd internal/obi && $(MAKE) docker-generate - @echo "✓ OBI eBPF artifacts generated successfully" - -# Check if OBI eBPF artifacts have been generated -.PHONY: check-obi-generated -check-obi-generated: - @if [ -d "internal/obi" ]; then \ - if ! find internal/obi -name "*_bpfel.go" | grep -q .; then \ - echo "Error: OBI eBPF artifacts not generated."; \ - echo "Run: make generate-obi"; \ - echo "This requires Docker to be available."; \ - exit 1; \ - fi \ - fi +.PHONY: prepare-obi +prepare-obi: + @./scripts/prepare-obi.sh "${DISTRIBUTIONS}" generate-msi: go ocb $(GO) run cmd/msi-generator/main.go -d "${DISTRIBUTIONS}" diff --git a/OBI_RECEIVER.md b/OBI_RECEIVER.md index 2ccc27e83..936a82d3a 100644 --- a/OBI_RECEIVER.md +++ b/OBI_RECEIVER.md @@ -1,24 +1,20 @@ # OBI Receiver Integration -This document explains how OBI (eBPF-based zero-code instrumentation) is integrated into the OpenTelemetry Collector distributions. +This document explains how OBI (eBPF-based zero-code instrumentation) is integrated into OpenTelemetry Collector distributions, why the integration is structured this way, and how to build and maintain it. ## Overview -The OBI receiver is included in the `otelcol-contrib` distribution as an external component. It is maintained in its own repository at [opentelemetry-ebpf-instrumentation](https://github.com/open-telemetry/opentelemetry-ebpf-instrumentation) and included here as a git submodule. +The OBI receiver is included in the `otelcol-contrib` distribution as an external component. It is maintained in its own repository at [opentelemetry-ebpf-instrumentation](https://github.com/open-telemetry/opentelemetry-ebpf-instrumentation). -## Architecture +To avoid git submodules, this repository prepares OBI from a tagged upstream source archive during builds. -### Why a Submodule? +## Architecture -OBI requires eBPF code generation (converting C code to Go) before the module can be built. The generated files are: -- **Not committed** to OBI's git repository (policy decision) -- **Platform-specific** (Linux kernel versions) -- **Large** (~10MB of binary objects and generated Go code) +OBI requires eBPF code generation (converting C code to Go) before Linux builds can compile OBI's eBPF-enabled paths. Generated files are not present in the upstream tag, so build preparation does the following: -Therefore, we need: -1. The OBI source code (git submodule at `internal/obi`) -2. A generation step before building (`make generate-obi`) -3. A replace directive to point the builder to the local submodule +1. Reads the OBI version from `distributions/otelcol-contrib/manifest.yaml` +2. Downloads the matching tagged source archive into `internal/obi-src` (untracked) +3. Runs `make docker-generate` in `internal/obi-src` when generated files are missing on Linux ### Integration Pattern @@ -30,215 +26,57 @@ receivers: import: go.opentelemetry.io/obi/collector replaces: - - go.opentelemetry.io/obi => ../../internal/obi + - go.opentelemetry.io/obi => ../../../internal/obi-src ``` -This uses the OpenTelemetry Collector Builder's `import:` field to directly reference the OBI collector package without needing a wrapper module. +This keeps the pinned module version in the manifest while using a locally prepared source tree for builds. ## Building with OBI ### Prerequisites -- Git with submodule support -- Docker (for eBPF code generation) +- Docker (required on Linux when generated OBI artifacts are missing) - Standard Go build tools -### First Time Setup - -```bash -# Clone with submodules -git clone --recurse-submodules https://github.com/open-telemetry/opentelemetry-collector-releases.git - -# Or initialize submodules if already cloned -git submodule update --init --recursive - -# Generate OBI eBPF artifacts (requires Docker, takes 2-5 minutes) -make generate-obi -``` - -### Building Distributions - -Once OBI artifacts are generated: +### Build Commands ```bash -# Build all distributions +# Build all distributions (includes OBI preparation for otelcol-contrib) make build -# Or build specific distribution +# Build only otelcol-contrib make build DISTRIBUTIONS="otelcol-contrib" ``` -The build process will automatically check that OBI artifacts exist before building. +`make build` and `make generate` call `scripts/prepare-obi.sh` automatically when `otelcol-contrib` is selected. ## CI/CD Integration -### Required Workflow Changes - -All CI workflows that build distributions must: - -1. **Initialize submodules:** - ```yaml - - uses: actions/checkout@v6 - with: - submodules: 'recursive' - ``` - -2. **Generate OBI artifacts (Linux only):** - ```yaml - - name: Generate OBI eBPF artifacts - if: runner.os == 'Linux' - run: make generate-obi - ``` - -3. **Build as normal:** - ```yaml - - name: Build distributions - run: make build - ``` +No submodule checkout is required. -### Build Time Impact - -- **Without caching:** Adds ~2-5 minutes to Linux builds -- **With caching:** Near-zero impact after first build -- **Submodule checkout:** Adds ~10-20 seconds - -### Artifact Caching - -To optimize CI builds, cache generated artifacts: - -```yaml -- name: Cache OBI artifacts - id: cache-obi - uses: actions/cache@v3 - with: - path: | - internal/obi/**/*_bpfel.go - internal/obi/**/*_bpfeb.go - internal/obi/**/*.o - key: obi-artifacts-${{ hashFiles('internal/obi/.git/HEAD') }} - -- name: Generate OBI eBPF artifacts - if: runner.os == 'Linux' && steps.cache-obi.outputs.cache-hit != 'true' - run: make generate-obi -``` +Linux CI jobs that build `otelcol-contrib` must still have Docker available for first-time OBI artifact generation. Caching `internal/obi-src` can reduce repeated generation costs. ## Maintenance -### Updating OBI Version - -1. **Update the submodule to a new commit:** - ```bash - cd internal/obi - git fetch - git checkout v0.6.0 # or specific commit - cd ../.. - git add internal/obi - ``` - -2. **Update the manifest version:** - ```bash - # Edit distributions/otelcol-contrib/manifest.yaml - # Change: gomod: go.opentelemetry.io/obi v0.5.0 - # To: gomod: go.opentelemetry.io/obi v0.6.0 - ``` - -3. **Regenerate artifacts:** - ```bash - make generate-obi - ``` +### Updating the OBI Version -4. **Test the build:** +1. Update `distributions/otelcol-contrib/manifest.yaml`: + - `gomod: go.opentelemetry.io/obi vX.Y.Z` +2. Build contrib: ```bash make build DISTRIBUTIONS="otelcol-contrib" ``` +3. Verify OBI preparation and generation complete successfully. -5. **Commit the changes:** - ```bash - git commit -m "Update OBI receiver to v0.6.0" - ``` - -### Troubleshooting - -#### Error: "OBI submodule not initialized" - -**Solution:** -```bash -git submodule update --init --recursive -``` - -#### Error: "OBI eBPF artifacts not generated" - -**Solution:** -```bash -make generate-obi -``` - -If generation fails, check: -- Docker is installed and running -- Docker can pull images -- You have sufficient disk space (generation needs ~500MB temp space) - -#### Slow Builds - -**Solution:** Implement CI artifact caching (see above) +No repository submodule updates are required. ## Platform Support -- ✅ **Linux (x86_64, arm64):** Full support with eBPF instrumentation -- ⚠️ **Other platforms:** Receiver included but runs as no-op (OBI uses build tags) - -The OBI code itself handles platform detection: -- `collector/factory_linux.go` - Full implementation -- `collector/factory_others.go` - No-op stub - -## Why Not in Collector-Contrib? - -The OBI receiver is an **external component** maintained in its own repository, not a **donated component**. According to collector-contrib documentation: - -> "you can just host it in your own repository as a Go module" - -The collector-releases repository is the appropriate place for: -- Packaging external components into distributions -- Managing build-time dependencies (like submodules) -- Distribution-specific configurations - -The collector-contrib repository is for: -- Components donated to the project -- Code maintained by the OpenTelemetry community -- Components that follow contrib's lifecycle - -## Future Improvements - -### Option 1: Pre-Generated Artifacts (Recommended) - -If OBI publishes pre-generated artifacts as GitHub release assets: - -1. Remove the submodule -2. Add a download step: `scripts/download-obi-artifacts.sh` -3. Remove the replace directive (use published module) - -This would eliminate all generation complexity. - -### Option 2: OBI Commits Artifacts - -If OBI changes policy and commits generated files to git: - -1. Remove the submodule -2. Remove generation targets from Makefile -3. Remove the replace directive -4. Reference OBI as a normal Go dependency - -This would be the simplest solution but requires OBI team approval. +- Linux: full OBI support (requires generated eBPF artifacts) +- Non-Linux: OBI's non-Linux code paths are used; Linux eBPF generation is skipped ## References -- **OBI Repository:** https://github.com/open-telemetry/opentelemetry-ebpf-instrumentation -- **OBI Collector Example:** https://github.com/open-telemetry/opentelemetry-ebpf-instrumentation/tree/main/examples/otel-collector -- **Collector Builder Docs:** https://opentelemetry.io/docs/collector/custom-collector/ - -## Questions? - -For questions about: -- **OBI functionality:** Open an issue in the OBI repository -- **Build integration:** Open an issue in this repository -- **Distribution inclusion:** Discuss in Collector SIG meetings +- OBI repository: https://github.com/open-telemetry/opentelemetry-ebpf-instrumentation +- OBI collector example: https://github.com/open-telemetry/opentelemetry-ebpf-instrumentation/tree/main/examples/otel-collector +- Collector builder docs: https://opentelemetry.io/docs/collector/custom-collector/ diff --git a/distributions/otelcol-contrib/manifest.yaml b/distributions/otelcol-contrib/manifest.yaml index dbfecbdd9..6cca14d21 100644 --- a/distributions/otelcol-contrib/manifest.yaml +++ b/distributions/otelcol-contrib/manifest.yaml @@ -131,7 +131,7 @@ processors: receivers: - gomod: go.opentelemetry.io/collector/receiver/nopreceiver v0.147.0 - gomod: go.opentelemetry.io/collector/receiver/otlpreceiver v0.147.0 - # OBI (eBPF Instrumentation) - requires submodule at internal/obi + # OBI (eBPF Instrumentation) - gomod: go.opentelemetry.io/obi v0.5.0 import: go.opentelemetry.io/obi/collector - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/activedirectorydsreceiver v0.147.0 @@ -264,6 +264,5 @@ providers: - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/confmap/provider/googlesecretmanagerprovider v0.147.0 replaces: - # OBI submodule - required for eBPF artifact generation - # Path is relative to distributions/otelcol-contrib/ - - go.opentelemetry.io/obi => ../../../internal/obi + # Prepared by scripts/prepare-obi.sh from a tagged upstream source archive. + - go.opentelemetry.io/obi => ../../../internal/obi-src diff --git a/internal/obi b/internal/obi deleted file mode 160000 index 754e7882f..000000000 --- a/internal/obi +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 754e7882f2d48302863570d3a64d0306352f1426 diff --git a/scripts/prepare-obi.sh b/scripts/prepare-obi.sh new file mode 100755 index 000000000..75d68164b --- /dev/null +++ b/scripts/prepare-obi.sh @@ -0,0 +1,76 @@ +#!/bin/bash + +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +REPO_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." &> /dev/null && pwd )" +DISTRIBUTIONS="${1:-}" +MANIFEST="${REPO_DIR}/distributions/otelcol-contrib/manifest.yaml" +OBI_DIR="${REPO_DIR}/internal/obi-src" + +needs_obi=false +if [[ "$DISTRIBUTIONS" == *"otelcol-contrib"* ]]; then + needs_obi=true +fi + +if [[ "${needs_obi}" != "true" ]]; then + exit 0 +fi + +if [[ ! -f "${MANIFEST}" ]]; then + echo "ERROR: OBI manifest not found at ${MANIFEST}" >&2 + exit 1 +fi + +obi_module_version="$( + awk '/- gomod: go\.opentelemetry\.io\/obi / {print $NF; exit}' "${MANIFEST}" +)" + +if [[ -z "${obi_module_version}" ]]; then + echo "ERROR: failed to resolve OBI version from ${MANIFEST}" >&2 + exit 1 +fi + +obi_archive_url="https://github.com/open-telemetry/opentelemetry-ebpf-instrumentation/archive/refs/tags/${obi_module_version}.tar.gz" +version_file="${OBI_DIR}/.obi-version" +prepared_version="" +if [[ -f "${version_file}" ]]; then + prepared_version="$(cat "${version_file}")" +fi + +if [[ "${prepared_version}" != "${obi_module_version}" ]]; then + tmpdir="$(mktemp -d)" + archive="${tmpdir}/obi-source.tar.gz" + trap 'rm -rf "${tmpdir}"' EXIT + + echo "Preparing OBI source ${obi_module_version} from ${obi_archive_url}" + curl --fail --show-error --location --retry 3 --retry-delay 1 \ + --output "${archive}" "${obi_archive_url}" + tar -xzf "${archive}" -C "${tmpdir}" + + extracted_dir="$(find "${tmpdir}" -mindepth 1 -maxdepth 1 -type d | head -n1)" + if [[ -z "${extracted_dir}" ]]; then + echo "ERROR: failed to unpack OBI archive ${obi_archive_url}" >&2 + exit 1 + fi + + rm -rf "${OBI_DIR}" + mv "${extracted_dir}" "${OBI_DIR}" + echo "${obi_module_version}" > "${version_file}" +fi + +# Linux builds compile OBI's eBPF-enabled paths and require generated files. +if [[ "$(uname -s)" != "Linux" ]]; then + exit 0 +fi + +if ! find "${OBI_DIR}" -name "*_bpfel.go" | grep -q .; then + if ! command -v docker > /dev/null 2>&1; then + echo "ERROR: docker is required to generate OBI eBPF artifacts on Linux." >&2 + exit 1 + fi + echo "Generating OBI eBPF artifacts (this may take a few minutes)..." + make -C "${OBI_DIR}" docker-generate +fi From cd950d3b13d2b8d26621a742df477003c9af22fa Mon Sep 17 00:00:00 2001 From: Tyler Yahn Date: Thu, 12 Mar 2026 17:05:33 -0700 Subject: [PATCH 3/3] Use OBI source released with generated files --- .github/actions/fetch-obi/action.yml | 116 ++++++++++++++++++++++ .github/workflows/base-ci-goreleaser.yaml | 4 + .github/workflows/base-release.yaml | 4 + .github/workflows/ci.yaml | 4 + .gitignore | 1 + Makefile | 8 ++ OBI_RECEIVER.md | 42 +++++--- scripts/prepare-obi.sh | 81 ++++++++------- 8 files changed, 211 insertions(+), 49 deletions(-) create mode 100644 .github/actions/fetch-obi/action.yml diff --git a/.github/actions/fetch-obi/action.yml b/.github/actions/fetch-obi/action.yml new file mode 100644 index 000000000..6ec880918 --- /dev/null +++ b/.github/actions/fetch-obi/action.yml @@ -0,0 +1,116 @@ +name: Fetch OBI source +description: | + Downloads the OBI (opentelemetry-ebpf-instrumentation) release tarball + (obi-vX.Y.Z-source-generated.tar.gz), which includes all pre-generated BPF + source files, verifies its SHA256 checksum, and extracts it to + internal/obi-src. The tarball is cached by OBI version so re-runs are instant + when the version has not changed. + + No BPF toolchain (Docker, clang, bpf2go) is required. + + The version is read from distributions/otelcol-contrib/manifest.yaml unless + obi-version is explicitly provided. + +inputs: + obi-version: + description: | + OBI version to fetch (e.g., v0.5.0). When empty (default), the version is + resolved from distributions/otelcol-contrib/manifest.yaml. + required: false + default: "" + +runs: + using: composite + steps: + - name: Resolve OBI version + shell: bash + run: | + VERSION="${{ inputs.obi-version }}" + if [[ -z "${VERSION}" ]]; then + VERSION=$(awk '/- gomod: go\.opentelemetry\.io\/obi / {print $NF; exit}' distributions/otelcol-contrib/manifest.yaml) + fi + if [[ -z "${VERSION}" ]]; then + echo "ERROR: failed to resolve OBI version from manifest.yaml" >&2 + exit 1 + fi + echo "OBI_VERSION=${VERSION}" >> "$GITHUB_ENV" + + - name: Cache OBI source + id: cache-obi + uses: actions/cache@v4 + with: + path: | + .local/obi-${{ env.OBI_VERSION }}-source-generated.tar.gz + internal/obi-src + key: obi-source-${{ env.OBI_VERSION }} + + - name: Fetch OBI source tarball (Linux/macOS) + if: steps.cache-obi.outputs.cache-hit != 'true' && runner.os != 'Windows' + shell: bash + run: | + TARBALL="obi-${OBI_VERSION}-source-generated.tar.gz" + BASE_URL="https://github.com/open-telemetry/opentelemetry-ebpf-instrumentation/releases/download/${OBI_VERSION}" + DEST="internal/obi-src" + + # Download tarball to .local/ — same location the script uses — so the + # version-keyed stamp file check in prepare-obi.sh finds it cached. + mkdir -p .local + curl -fL -o ".local/${TARBALL}" "${BASE_URL}/${TARBALL}" + + # Verify checksum via the upstream SHA256SUMS release asset. + # Use uname to distinguish Linux (GNU sha256sum) from macOS (BSD sha256sum + # lacks --check; use perl shasum instead). + if [[ "$(uname -s)" == "Linux" ]]; then + curl -fsSL "${BASE_URL}/SHA256SUMS" | grep -F "${TARBALL}" | (cd .local && sha256sum --check) + else + curl -fsSL "${BASE_URL}/SHA256SUMS" | grep -F "${TARBALL}" | (cd .local && shasum -a 256 --check) + fi + + # Extract verified tarball. + mkdir -p "${DEST}" + tar xzf ".local/${TARBALL}" --strip-components=1 -C "${DEST}" + + # Create the version-keyed stamp file so prepare-obi.sh exits immediately + # on subsequent Make invocations (matches OBI_STAMP in the Makefile). + touch "${DEST}/.obi-${OBI_VERSION}" + + - name: Fetch OBI source tarball (Windows) + if: steps.cache-obi.outputs.cache-hit != 'true' && runner.os == 'Windows' + shell: pwsh + run: | + $Version = $env:OBI_VERSION + $Tarball = "obi-${Version}-source-generated.tar.gz" + $BaseUrl = "https://github.com/open-telemetry/opentelemetry-ebpf-instrumentation/releases/download/${Version}" + $Dest = "internal\obi-src" + + # Download tarball to .local\ — same location the script uses. + New-Item -ItemType Directory -Force -Path ".local" | Out-Null + $LocalTarball = ".local\$Tarball" + curl.exe -fL -o $LocalTarball "${BaseUrl}/${Tarball}" + + # Download SHA256SUMS and extract the expected hash for our tarball. + $SumsFile = ".local\SHA256SUMS.tmp" + curl.exe -fsSL -o $SumsFile "${BaseUrl}/SHA256SUMS" + $Expected = (Get-Content $SumsFile | Where-Object { $_ -match [regex]::Escape($Tarball) } | Select-Object -First 1) + Remove-Item $SumsFile + if (-not $Expected) { + Write-Error "ERROR: ${Tarball} not found in SHA256SUMS" + exit 1 + } + $Expected = $Expected.Split(' ')[0].Trim() + + # Verify hash. + $Actual = (Get-FileHash $LocalTarball -Algorithm SHA256).Hash.ToLower() + if ($Actual -ne $Expected) { + Write-Error "ERROR: SHA256 checksum verification failed for ${Tarball}`n Expected: ${Expected}`n Actual: ${Actual}" + exit 1 + } + Write-Host "SHA256 verified: ${Tarball}" + + # Extract verified tarball. + New-Item -ItemType Directory -Force -Path $Dest | Out-Null + tar xzf $LocalTarball --strip-components=1 -C $Dest + + # Create the version-keyed stamp file so prepare-obi.sh exits immediately. + New-Item -ItemType File -Force -Path "$Dest\.obi-$Version" | Out-Null + diff --git a/.github/workflows/base-ci-goreleaser.yaml b/.github/workflows/base-ci-goreleaser.yaml index 548431346..ae3937600 100644 --- a/.github/workflows/base-ci-goreleaser.yaml +++ b/.github/workflows/base-ci-goreleaser.yaml @@ -114,6 +114,10 @@ jobs: go-version: "~1.26.0" check-latest: true + - name: Fetch OBI source + if: inputs.distribution == 'otelcol-contrib' && inputs.nightly != true + uses: ./.github/actions/fetch-obi + - name: Create artifacts directory to store build artifacts if: inputs.distribution == 'otelcol-contrib' run: mkdir -p distributions/otelcol-contrib/artifacts diff --git a/.github/workflows/base-release.yaml b/.github/workflows/base-release.yaml index cf6e422a9..9a39cf999 100644 --- a/.github/workflows/base-release.yaml +++ b/.github/workflows/base-release.yaml @@ -108,6 +108,10 @@ jobs: run: | echo "NIGHTLY_FLAG=--nightly" >> "$GITHUB_OUTPUT" + - name: Fetch OBI source + if: inputs.distribution == 'otelcol-contrib' + uses: ./.github/actions/fetch-obi + - name: Generate distribution sources run: make generate-sources env: diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 393bd9b7c..e2e878db1 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -39,6 +39,10 @@ jobs: go-version: "~1.26.0" check-latest: true + - name: Fetch OBI source + if: matrix.distro == 'otelcol-contrib' + uses: ./.github/actions/fetch-obi + - name: Tidy go.mod files run: go mod tidy diff --git a/.gitignore b/.gitignore index d73d6fb61..c8c631845 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,4 @@ dist/ distributions/*/windows-installer.wxs internal/obi/ internal/obi-src/ +.local/ diff --git a/Makefile b/Makefile index aca1a3b28..165d614ba 100644 --- a/Makefile +++ b/Makefile @@ -36,6 +36,14 @@ generate-goreleaser: go generate-sources: go ocb generate-msi prepare-obi @./scripts/build.sh -d "${DISTRIBUTIONS}" -s true -b ${OTELCOL_BUILDER} +# OBI (opentelemetry-ebpf-instrumentation) configuration. +# The version is kept here and in distributions/otelcol-contrib/manifest.yaml; +# they must stay in sync. The stamp file is version-keyed so that changing +# OBI_VERSION automatically triggers a re-download on the next build. +OBI_VERSION := $(shell awk '/- gomod: go\.opentelemetry\.io\/obi / { print $$NF; exit }' $(SRC_ROOT)/distributions/otelcol-contrib/manifest.yaml) +OBI_DIR := $(SRC_ROOT)/internal/obi-src +OBI_STAMP := $(OBI_DIR)/.obi-$(OBI_VERSION) + .PHONY: prepare-obi prepare-obi: @./scripts/prepare-obi.sh "${DISTRIBUTIONS}" diff --git a/OBI_RECEIVER.md b/OBI_RECEIVER.md index 936a82d3a..8a89c4bfa 100644 --- a/OBI_RECEIVER.md +++ b/OBI_RECEIVER.md @@ -6,15 +6,19 @@ This document explains how OBI (eBPF-based zero-code instrumentation) is integra The OBI receiver is included in the `otelcol-contrib` distribution as an external component. It is maintained in its own repository at [opentelemetry-ebpf-instrumentation](https://github.com/open-telemetry/opentelemetry-ebpf-instrumentation). -To avoid git submodules, this repository prepares OBI from a tagged upstream source archive during builds. +To avoid git submodules, this repository downloads the OBI release tarball (`obi-vX.Y.Z-source-generated.tar.gz`) during builds. This tarball is published alongside each OBI release and already contains all pre-generated BPF source files. ## Architecture -OBI requires eBPF code generation (converting C code to Go) before Linux builds can compile OBI's eBPF-enabled paths. Generated files are not present in the upstream tag, so build preparation does the following: +Build preparation does the following: 1. Reads the OBI version from `distributions/otelcol-contrib/manifest.yaml` -2. Downloads the matching tagged source archive into `internal/obi-src` (untracked) -3. Runs `make docker-generate` in `internal/obi-src` when generated files are missing on Linux +2. Downloads `obi-vX.Y.Z-source-generated.tar.gz` from the OBI GitHub release +3. Verifies the SHA256 checksum against the `SHA256SUMS` release asset +4. Extracts the tarball to `internal/obi-src` (untracked) +5. Creates a version-keyed stamp file (`internal/obi-src/.obi-vX.Y.Z`) so subsequent builds skip the download + +No BPF toolchain (Docker, clang, bpf2go) is required. The pre-generated files are included in the `source-generated` tarball. ### Integration Pattern @@ -35,8 +39,7 @@ This keeps the pinned module version in the manifest while using a locally prepa ### Prerequisites -- Docker (required on Linux when generated OBI artifacts are missing) -- Standard Go build tools +- Standard Go build tools (no Docker required) ### Build Commands @@ -52,9 +55,19 @@ make build DISTRIBUTIONS="otelcol-contrib" ## CI/CD Integration -No submodule checkout is required. +CI jobs that build `otelcol-contrib` should run the `.github/actions/fetch-obi` composite action **before** any `make` target that triggers source generation. The action: + +1. Checks the Actions cache for the OBI version (cache key: `obi-source-vX.Y.Z`) +2. On a cache miss: downloads and verifies the tarball, extracts it, and writes the stamp file +3. On a cache hit: restores from cache (stamp file already present → `prepare-obi.sh` exits immediately) -Linux CI jobs that build `otelcol-contrib` must still have Docker available for first-time OBI artifact generation. Caching `internal/obi-src` can reduce repeated generation costs. +This means no Docker or network access is needed during the actual `make` build step. + +```yaml +- name: Fetch OBI source + if: inputs.distribution == 'otelcol-contrib' + uses: ./.github/actions/fetch-obi +``` ## Maintenance @@ -62,21 +75,24 @@ Linux CI jobs that build `otelcol-contrib` must still have Docker available for 1. Update `distributions/otelcol-contrib/manifest.yaml`: - `gomod: go.opentelemetry.io/obi vX.Y.Z` -2. Build contrib: +2. Build contrib to verify: ```bash make build DISTRIBUTIONS="otelcol-contrib" ``` -3. Verify OBI preparation and generation complete successfully. +3. Commit both the manifest change and any `go.sum` updates. -No repository submodule updates are required. +No repository submodule updates are required. The CI cache is invalidated automatically because the cache key is version-keyed. ## Platform Support -- Linux: full OBI support (requires generated eBPF artifacts) -- Non-Linux: OBI's non-Linux code paths are used; Linux eBPF generation is skipped +- Linux amd64/arm64: full OBI support (eBPF receiver active) +- All other platforms (Linux ppc64le/s390x/riscv64, Windows, macOS): OBI's non-eBPF stubs are compiled; the receiver registers but does not activate eBPF collection + +The `go.opentelemetry.io/obi/collector` package uses build tags (`//go:build linux && (amd64 || arm64)`) to select the appropriate implementation. No cross-compilation guards are needed in this repository. ## References - OBI repository: https://github.com/open-telemetry/opentelemetry-ebpf-instrumentation - OBI collector example: https://github.com/open-telemetry/opentelemetry-ebpf-instrumentation/tree/main/examples/otel-collector - Collector builder docs: https://opentelemetry.io/docs/collector/custom-collector/ + diff --git a/scripts/prepare-obi.sh b/scripts/prepare-obi.sh index 75d68164b..b3e1eafd9 100755 --- a/scripts/prepare-obi.sh +++ b/scripts/prepare-obi.sh @@ -3,6 +3,15 @@ # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 +# Downloads the OBI (opentelemetry-ebpf-instrumentation) release tarball +# (obi-vX.Y.Z-source-generated.tar.gz), which includes all pre-generated BPF +# source files, verifies its SHA256 checksum, and extracts it to internal/obi-src. +# +# No BPF toolchain (Docker, clang, bpf2go) is required. +# +# The version is read from distributions/otelcol-contrib/manifest.yaml unless +# OBI_VERSION is already set in the environment (e.g. by the CI composite action). + set -euo pipefail REPO_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." &> /dev/null && pwd )" @@ -24,53 +33,53 @@ if [[ ! -f "${MANIFEST}" ]]; then exit 1 fi -obi_module_version="$( +OBI_VERSION="${OBI_VERSION:-$( awk '/- gomod: go\.opentelemetry\.io\/obi / {print $NF; exit}' "${MANIFEST}" -)" +)}" -if [[ -z "${obi_module_version}" ]]; then +if [[ -z "${OBI_VERSION}" ]]; then echo "ERROR: failed to resolve OBI version from ${MANIFEST}" >&2 exit 1 fi -obi_archive_url="https://github.com/open-telemetry/opentelemetry-ebpf-instrumentation/archive/refs/tags/${obi_module_version}.tar.gz" -version_file="${OBI_DIR}/.obi-version" -prepared_version="" -if [[ -f "${version_file}" ]]; then - prepared_version="$(cat "${version_file}")" +TARBALL="obi-${OBI_VERSION}-source-generated.tar.gz" +BASE_URL="https://github.com/open-telemetry/opentelemetry-ebpf-instrumentation/releases/download/${OBI_VERSION}" +TARBALL_CACHE="${REPO_DIR}/.local/${TARBALL}" +OBI_STAMP="${OBI_DIR}/.obi-${OBI_VERSION}" + +# Fast path: version-keyed stamp file already exists → nothing to do. +if [[ -f "${OBI_STAMP}" ]]; then + echo "OBI ${OBI_VERSION} already prepared at ${OBI_DIR}" + exit 0 fi -if [[ "${prepared_version}" != "${obi_module_version}" ]]; then - tmpdir="$(mktemp -d)" - archive="${tmpdir}/obi-source.tar.gz" - trap 'rm -rf "${tmpdir}"' EXIT +echo "Fetching OBI ${OBI_VERSION} source-generated tarball..." - echo "Preparing OBI source ${obi_module_version} from ${obi_archive_url}" +# Download the tarball to .local/ if it is not already cached there. +mkdir -p "${REPO_DIR}/.local" +if [[ ! -f "${TARBALL_CACHE}" ]]; then curl --fail --show-error --location --retry 3 --retry-delay 1 \ - --output "${archive}" "${obi_archive_url}" - tar -xzf "${archive}" -C "${tmpdir}" - - extracted_dir="$(find "${tmpdir}" -mindepth 1 -maxdepth 1 -type d | head -n1)" - if [[ -z "${extracted_dir}" ]]; then - echo "ERROR: failed to unpack OBI archive ${obi_archive_url}" >&2 - exit 1 - fi - - rm -rf "${OBI_DIR}" - mv "${extracted_dir}" "${OBI_DIR}" - echo "${obi_module_version}" > "${version_file}" + --output "${TARBALL_CACHE}" "${BASE_URL}/${TARBALL}" fi -# Linux builds compile OBI's eBPF-enabled paths and require generated files. -if [[ "$(uname -s)" != "Linux" ]]; then - exit 0 +# Verify checksum against the upstream SHA256SUMS release asset. +echo "Verifying OBI ${OBI_VERSION} tarball checksum..." +if [[ "$(uname -s)" == "Linux" ]]; then + curl -fsSL "${BASE_URL}/SHA256SUMS" | grep -F "${TARBALL}" \ + | (cd "${REPO_DIR}/.local" && sha256sum --check) \ + || { rm -f "${TARBALL_CACHE}"; echo "ERROR: checksum verification failed." >&2; exit 1; } +else + curl -fsSL "${BASE_URL}/SHA256SUMS" | grep -F "${TARBALL}" \ + | (cd "${REPO_DIR}/.local" && shasum -a 256 --check) \ + || { rm -f "${TARBALL_CACHE}"; echo "ERROR: checksum verification failed." >&2; exit 1; } fi -if ! find "${OBI_DIR}" -name "*_bpfel.go" | grep -q .; then - if ! command -v docker > /dev/null 2>&1; then - echo "ERROR: docker is required to generate OBI eBPF artifacts on Linux." >&2 - exit 1 - fi - echo "Generating OBI eBPF artifacts (this may take a few minutes)..." - make -C "${OBI_DIR}" docker-generate -fi +# Extract to OBI_DIR. +rm -rf "${OBI_DIR}" +mkdir -p "${OBI_DIR}" +tar xzf "${TARBALL_CACHE}" --strip-components=1 -C "${OBI_DIR}" + +# Version-keyed stamp file: signals that this OBI version is ready and lets +# subsequent Make invocations (and the CI composite action) skip the download. +touch "${OBI_STAMP}" +echo "OBI ${OBI_VERSION} source prepared at ${OBI_DIR}"