diff --git a/.github/actions/fetch-obi/action.yml b/.github/actions/fetch-obi/action.yml new file mode 100644 index 000000000..6ec880918 --- /dev/null +++ b/.github/actions/fetch-obi/action.yml @@ -0,0 +1,116 @@ +name: Fetch OBI source +description: | + Downloads the OBI (opentelemetry-ebpf-instrumentation) release tarball + (obi-vX.Y.Z-source-generated.tar.gz), which includes all pre-generated BPF + source files, verifies its SHA256 checksum, and extracts it to + internal/obi-src. The tarball is cached by OBI version so re-runs are instant + when the version has not changed. + + No BPF toolchain (Docker, clang, bpf2go) is required. + + The version is read from distributions/otelcol-contrib/manifest.yaml unless + obi-version is explicitly provided. + +inputs: + obi-version: + description: | + OBI version to fetch (e.g., v0.5.0). When empty (default), the version is + resolved from distributions/otelcol-contrib/manifest.yaml. + required: false + default: "" + +runs: + using: composite + steps: + - name: Resolve OBI version + shell: bash + run: | + VERSION="${{ inputs.obi-version }}" + if [[ -z "${VERSION}" ]]; then + VERSION=$(awk '/- gomod: go\.opentelemetry\.io\/obi / {print $NF; exit}' distributions/otelcol-contrib/manifest.yaml) + fi + if [[ -z "${VERSION}" ]]; then + echo "ERROR: failed to resolve OBI version from manifest.yaml" >&2 + exit 1 + fi + echo "OBI_VERSION=${VERSION}" >> "$GITHUB_ENV" + + - name: Cache OBI source + id: cache-obi + uses: actions/cache@v4 + with: + path: | + .local/obi-${{ env.OBI_VERSION }}-source-generated.tar.gz + internal/obi-src + key: obi-source-${{ env.OBI_VERSION }} + + - name: Fetch OBI source tarball (Linux/macOS) + if: steps.cache-obi.outputs.cache-hit != 'true' && runner.os != 'Windows' + shell: bash + run: | + TARBALL="obi-${OBI_VERSION}-source-generated.tar.gz" + BASE_URL="https://github.com/open-telemetry/opentelemetry-ebpf-instrumentation/releases/download/${OBI_VERSION}" + DEST="internal/obi-src" + + # Download tarball to .local/ — same location the script uses — so the + # version-keyed stamp file check in prepare-obi.sh finds it cached. + mkdir -p .local + curl -fL -o ".local/${TARBALL}" "${BASE_URL}/${TARBALL}" + + # Verify checksum via the upstream SHA256SUMS release asset. + # Use uname to distinguish Linux (GNU sha256sum) from macOS (BSD sha256sum + # lacks --check; use perl shasum instead). + if [[ "$(uname -s)" == "Linux" ]]; then + curl -fsSL "${BASE_URL}/SHA256SUMS" | grep -F "${TARBALL}" | (cd .local && sha256sum --check) + else + curl -fsSL "${BASE_URL}/SHA256SUMS" | grep -F "${TARBALL}" | (cd .local && shasum -a 256 --check) + fi + + # Extract verified tarball. + mkdir -p "${DEST}" + tar xzf ".local/${TARBALL}" --strip-components=1 -C "${DEST}" + + # Create the version-keyed stamp file so prepare-obi.sh exits immediately + # on subsequent Make invocations (matches OBI_STAMP in the Makefile). + touch "${DEST}/.obi-${OBI_VERSION}" + + - name: Fetch OBI source tarball (Windows) + if: steps.cache-obi.outputs.cache-hit != 'true' && runner.os == 'Windows' + shell: pwsh + run: | + $Version = $env:OBI_VERSION + $Tarball = "obi-${Version}-source-generated.tar.gz" + $BaseUrl = "https://github.com/open-telemetry/opentelemetry-ebpf-instrumentation/releases/download/${Version}" + $Dest = "internal\obi-src" + + # Download tarball to .local\ — same location the script uses. + New-Item -ItemType Directory -Force -Path ".local" | Out-Null + $LocalTarball = ".local\$Tarball" + curl.exe -fL -o $LocalTarball "${BaseUrl}/${Tarball}" + + # Download SHA256SUMS and extract the expected hash for our tarball. + $SumsFile = ".local\SHA256SUMS.tmp" + curl.exe -fsSL -o $SumsFile "${BaseUrl}/SHA256SUMS" + $Expected = (Get-Content $SumsFile | Where-Object { $_ -match [regex]::Escape($Tarball) } | Select-Object -First 1) + Remove-Item $SumsFile + if (-not $Expected) { + Write-Error "ERROR: ${Tarball} not found in SHA256SUMS" + exit 1 + } + $Expected = $Expected.Split(' ')[0].Trim() + + # Verify hash. + $Actual = (Get-FileHash $LocalTarball -Algorithm SHA256).Hash.ToLower() + if ($Actual -ne $Expected) { + Write-Error "ERROR: SHA256 checksum verification failed for ${Tarball}`n Expected: ${Expected}`n Actual: ${Actual}" + exit 1 + } + Write-Host "SHA256 verified: ${Tarball}" + + # Extract verified tarball. + New-Item -ItemType Directory -Force -Path $Dest | Out-Null + tar xzf $LocalTarball --strip-components=1 -C $Dest + + # Create the version-keyed stamp file so prepare-obi.sh exits immediately. + New-Item -ItemType File -Force -Path "$Dest\.obi-$Version" | Out-Null + diff --git a/.github/workflows/base-ci-goreleaser.yaml b/.github/workflows/base-ci-goreleaser.yaml index a23ff4b76..a5506ad9a 100644 --- a/.github/workflows/base-ci-goreleaser.yaml +++ b/.github/workflows/base-ci-goreleaser.yaml @@ -119,6 +119,10 @@ jobs: go-version: "~1.26.0" check-latest: true + - name: Fetch OBI source + if: inputs.distribution == 'otelcol-contrib' && inputs.nightly != true + uses: ./.github/actions/fetch-obi + - name: Create artifacts directory to store build artifacts if: inputs.distribution == 'otelcol-contrib' run: mkdir -p distributions/otelcol-contrib/artifacts diff --git a/.github/workflows/base-release.yaml b/.github/workflows/base-release.yaml index 0a169c2e1..8a4c49792 100644 --- a/.github/workflows/base-release.yaml +++ b/.github/workflows/base-release.yaml @@ -108,6 +108,10 @@ jobs: run: | echo "NIGHTLY_FLAG=--nightly" >> "$GITHUB_OUTPUT" + - name: Fetch OBI source + if: inputs.distribution == 'otelcol-contrib' + uses: ./.github/actions/fetch-obi + - name: Generate distribution sources run: make generate-sources env: diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index bfe4c5013..8ca56efe6 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -39,6 +39,10 @@ jobs: go-version: "~1.26.0" check-latest: true + - name: Fetch OBI source + if: matrix.distro == 'otelcol-contrib' + uses: ./.github/actions/fetch-obi + - name: Tidy go.mod files run: go mod tidy diff --git a/.gitignore b/.gitignore index 9d76ce30e..c8c631845 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,7 @@ dist/ .tools .DS_Store .zed +distributions/*/windows-installer.wxs +internal/obi/ +internal/obi-src/ +.local/ diff --git a/Makefile b/Makefile index 731cdb641..28d1f0c7d 100644 --- a/Makefile +++ b/Makefile @@ -25,7 +25,7 @@ BINARIES ?= "builder,opampsupervisor" ci: check build check: ensure-goreleaser-up-to-date validate-components validate-version-consistency -build: go ocb +build: go ocb prepare-obi @./scripts/build.sh -d "${DISTRIBUTIONS}" -b ${OTELCOL_BUILDER} generate: generate-sources generate-goreleaser @@ -33,9 +33,21 @@ generate: generate-sources generate-goreleaser generate-goreleaser: go @./scripts/generate-goreleaser.sh -d "${DISTRIBUTIONS}" -b "${BINARIES}" -g ${GO} -generate-sources: go ocb generate-msi +generate-sources: go ocb generate-msi prepare-obi @./scripts/build.sh -d "${DISTRIBUTIONS}" -s true -b ${OTELCOL_BUILDER} +# OBI (opentelemetry-ebpf-instrumentation) configuration. +# The version is kept here and in distributions/otelcol-contrib/manifest.yaml; +# they must stay in sync. The stamp file is version-keyed so that changing +# OBI_VERSION automatically triggers a re-download on the next build. +OBI_VERSION := $(shell awk '/- gomod: go\.opentelemetry\.io\/obi / { print $$NF; exit }' $(SRC_ROOT)/distributions/otelcol-contrib/manifest.yaml) +OBI_DIR := $(SRC_ROOT)/internal/obi-src +OBI_STAMP := $(OBI_DIR)/.obi-$(OBI_VERSION) + +.PHONY: prepare-obi +prepare-obi: + @./scripts/prepare-obi.sh "${DISTRIBUTIONS}" + generate-msi: go ocb $(GO) run cmd/msi-generator/main.go -d "${DISTRIBUTIONS}" diff --git a/OBI_RECEIVER.md b/OBI_RECEIVER.md new file mode 100644 index 000000000..8a89c4bfa --- /dev/null +++ b/OBI_RECEIVER.md @@ -0,0 +1,98 @@ +# OBI Receiver Integration + +This document explains how OBI (eBPF-based zero-code instrumentation) is integrated into OpenTelemetry Collector distributions, why the integration is structured this way, and how to build and maintain it. + +## Overview + +The OBI receiver is included in the `otelcol-contrib` distribution as an external component. It is maintained in its own repository at [opentelemetry-ebpf-instrumentation](https://github.com/open-telemetry/opentelemetry-ebpf-instrumentation). + +To avoid git submodules, this repository downloads the OBI release tarball (`obi-vX.Y.Z-source-generated.tar.gz`) during builds. This tarball is published alongside each OBI release and already contains all pre-generated BPF source files. + +## Architecture + +Build preparation does the following: + +1. Reads the OBI version from `distributions/otelcol-contrib/manifest.yaml` +2. Downloads `obi-vX.Y.Z-source-generated.tar.gz` from the OBI GitHub release +3. Verifies the SHA256 checksum against the `SHA256SUMS` release asset +4. Extracts the tarball to `internal/obi-src` (untracked) +5. Creates a version-keyed stamp file (`internal/obi-src/.obi-vX.Y.Z`) so subsequent builds skip the download + +No BPF toolchain (Docker, clang, bpf2go) is required. The pre-generated files are included in the `source-generated` tarball. + +### Integration Pattern + +The `otelcol-contrib` distribution manifest includes: + +```yaml +receivers: + - gomod: go.opentelemetry.io/obi v0.5.0 + import: go.opentelemetry.io/obi/collector + +replaces: + - go.opentelemetry.io/obi => ../../../internal/obi-src +``` + +This keeps the pinned module version in the manifest while using a locally prepared source tree for builds. + +## Building with OBI + +### Prerequisites + +- Standard Go build tools (no Docker required) + +### Build Commands + +```bash +# Build all distributions (includes OBI preparation for otelcol-contrib) +make build + +# Build only otelcol-contrib +make build DISTRIBUTIONS="otelcol-contrib" +``` + +`make build` and `make generate` call `scripts/prepare-obi.sh` automatically when `otelcol-contrib` is selected. + +## CI/CD Integration + +CI jobs that build `otelcol-contrib` should run the `.github/actions/fetch-obi` composite action **before** any `make` target that triggers source generation. The action: + +1. Checks the Actions cache for the OBI version (cache key: `obi-source-vX.Y.Z`) +2. On a cache miss: downloads and verifies the tarball, extracts it, and writes the stamp file +3. On a cache hit: restores from cache (stamp file already present → `prepare-obi.sh` exits immediately) + +This means no Docker or network access is needed during the actual `make` build step. + +```yaml +- name: Fetch OBI source + if: inputs.distribution == 'otelcol-contrib' + uses: ./.github/actions/fetch-obi +``` + +## Maintenance + +### Updating the OBI Version + +1. Update `distributions/otelcol-contrib/manifest.yaml`: + - `gomod: go.opentelemetry.io/obi vX.Y.Z` +2. Build contrib to verify: + ```bash + make build DISTRIBUTIONS="otelcol-contrib" + ``` +3. Commit both the manifest change and any `go.sum` updates. + +No repository submodule updates are required. The CI cache is invalidated automatically because the cache key is version-keyed. + +## Platform Support + +- Linux amd64/arm64: full OBI support (eBPF receiver active) +- All other platforms (Linux ppc64le/s390x/riscv64, Windows, macOS): OBI's non-eBPF stubs are compiled; the receiver registers but does not activate eBPF collection + +The `go.opentelemetry.io/obi/collector` package uses build tags (`//go:build linux && (amd64 || arm64)`) to select the appropriate implementation. No cross-compilation guards are needed in this repository. + +## References + +- OBI repository: https://github.com/open-telemetry/opentelemetry-ebpf-instrumentation +- OBI collector example: https://github.com/open-telemetry/opentelemetry-ebpf-instrumentation/tree/main/examples/otel-collector +- Collector builder docs: https://opentelemetry.io/docs/collector/custom-collector/ + diff --git a/distributions/otelcol-contrib/manifest.yaml b/distributions/otelcol-contrib/manifest.yaml index b5b48d685..93eae35c3 100644 --- a/distributions/otelcol-contrib/manifest.yaml +++ b/distributions/otelcol-contrib/manifest.yaml @@ -133,6 +133,9 @@ processors: receivers: - gomod: go.opentelemetry.io/collector/receiver/nopreceiver v0.151.0 - gomod: go.opentelemetry.io/collector/receiver/otlpreceiver v0.151.0 + # OBI (eBPF Instrumentation) + - gomod: go.opentelemetry.io/obi v0.5.0 + import: go.opentelemetry.io/obi/collector - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/activedirectorydsreceiver v0.151.0 - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/aerospikereceiver v0.151.0 - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/apachereceiver v0.151.0 @@ -264,3 +267,7 @@ providers: - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/confmap/provider/s3provider v0.151.0 - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/confmap/provider/secretsmanagerprovider v0.151.0 - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/confmap/provider/googlesecretmanagerprovider v0.151.0 + +replaces: + # Prepared by scripts/prepare-obi.sh from a tagged upstream source archive. + - go.opentelemetry.io/obi => ../../../internal/obi-src diff --git a/scripts/prepare-obi.sh b/scripts/prepare-obi.sh new file mode 100755 index 000000000..b3e1eafd9 --- /dev/null +++ b/scripts/prepare-obi.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +# Downloads the OBI (opentelemetry-ebpf-instrumentation) release tarball +# (obi-vX.Y.Z-source-generated.tar.gz), which includes all pre-generated BPF +# source files, verifies its SHA256 checksum, and extracts it to internal/obi-src. +# +# No BPF toolchain (Docker, clang, bpf2go) is required. +# +# The version is read from distributions/otelcol-contrib/manifest.yaml unless +# OBI_VERSION is already set in the environment (e.g. by the CI composite action). + +set -euo pipefail + +REPO_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." &> /dev/null && pwd )" +DISTRIBUTIONS="${1:-}" +MANIFEST="${REPO_DIR}/distributions/otelcol-contrib/manifest.yaml" +OBI_DIR="${REPO_DIR}/internal/obi-src" + +needs_obi=false +if [[ "$DISTRIBUTIONS" == *"otelcol-contrib"* ]]; then + needs_obi=true +fi + +if [[ "${needs_obi}" != "true" ]]; then + exit 0 +fi + +if [[ ! -f "${MANIFEST}" ]]; then + echo "ERROR: OBI manifest not found at ${MANIFEST}" >&2 + exit 1 +fi + +OBI_VERSION="${OBI_VERSION:-$( + awk '/- gomod: go\.opentelemetry\.io\/obi / {print $NF; exit}' "${MANIFEST}" +)}" + +if [[ -z "${OBI_VERSION}" ]]; then + echo "ERROR: failed to resolve OBI version from ${MANIFEST}" >&2 + exit 1 +fi + +TARBALL="obi-${OBI_VERSION}-source-generated.tar.gz" +BASE_URL="https://github.com/open-telemetry/opentelemetry-ebpf-instrumentation/releases/download/${OBI_VERSION}" +TARBALL_CACHE="${REPO_DIR}/.local/${TARBALL}" +OBI_STAMP="${OBI_DIR}/.obi-${OBI_VERSION}" + +# Fast path: version-keyed stamp file already exists → nothing to do. +if [[ -f "${OBI_STAMP}" ]]; then + echo "OBI ${OBI_VERSION} already prepared at ${OBI_DIR}" + exit 0 +fi + +echo "Fetching OBI ${OBI_VERSION} source-generated tarball..." + +# Download the tarball to .local/ if it is not already cached there. +mkdir -p "${REPO_DIR}/.local" +if [[ ! -f "${TARBALL_CACHE}" ]]; then + curl --fail --show-error --location --retry 3 --retry-delay 1 \ + --output "${TARBALL_CACHE}" "${BASE_URL}/${TARBALL}" +fi + +# Verify checksum against the upstream SHA256SUMS release asset. +echo "Verifying OBI ${OBI_VERSION} tarball checksum..." +if [[ "$(uname -s)" == "Linux" ]]; then + curl -fsSL "${BASE_URL}/SHA256SUMS" | grep -F "${TARBALL}" \ + | (cd "${REPO_DIR}/.local" && sha256sum --check) \ + || { rm -f "${TARBALL_CACHE}"; echo "ERROR: checksum verification failed." >&2; exit 1; } +else + curl -fsSL "${BASE_URL}/SHA256SUMS" | grep -F "${TARBALL}" \ + | (cd "${REPO_DIR}/.local" && shasum -a 256 --check) \ + || { rm -f "${TARBALL_CACHE}"; echo "ERROR: checksum verification failed." >&2; exit 1; } +fi + +# Extract to OBI_DIR. +rm -rf "${OBI_DIR}" +mkdir -p "${OBI_DIR}" +tar xzf "${TARBALL_CACHE}" --strip-components=1 -C "${OBI_DIR}" + +# Version-keyed stamp file: signals that this OBI version is ready and lets +# subsequent Make invocations (and the CI composite action) skip the download. +touch "${OBI_STAMP}" +echo "OBI ${OBI_VERSION} source prepared at ${OBI_DIR}"