go labels dd

.github/renovate.json5

@@ -0,0 +1,21 @@
+{
+  "$schema": "https://docs.renovatebot.com/renovate-schema.json",
+  "extends": [
+    "config:best-practices",
+    "helpers:pinGitHubActionDigestsToSemver"
+  ],
+  "packageRules": [
+    {
+      "groupName": "all patch versions",
+      "matchUpdateTypes": ["patch"],
+      "schedule": ["before 8am every weekday"]
+    },
+    {
+      "matchUpdateTypes": ["minor", "major"],
+      "schedule": ["before 8am on Monday"]
+    }
+  ],
+  "labels": [
+    "dependencies"
+  ]
+}

.github/workflows/unit-test-on-pull-request.yml

@@ -138,7 +138,6 @@ jobs:
           # https://github.com/cilium/ci-kernels/pkgs/container/ci-kernels/versions?filters%5Bversion_type%5D=tagged
 
           # AMD64
-          - { target_arch: amd64, kernel: 4.19.314 }
           - { target_arch: amd64, kernel: 5.4.276 }
           - { target_arch: amd64, kernel: 5.10.217 }
           - { target_arch: amd64, kernel: 5.15.159 }

CONTRIBUTING.md

@@ -20,7 +20,7 @@ slack channel for discussions and questions.
 
 ## Pre-requisites
 
-- Linux (4.19+ for x86-64, 5.5+ for ARM64) with eBPF enabled (the profiler currently only runs on Linux)
+- Linux (5.4+ for x86-64, 5.5+ for ARM64) with eBPF enabled (the profiler currently only runs on Linux)
 - Go as specified in [go.mod](https://github.com/open-telemetry/opentelemetry-ebpf-profiler/blob/main/go.mod)
 - docker
 - Rust as specified in [Cargo.toml](https://github.com/open-telemetry/opentelemetry-ebpf-profiler/blob/main/Cargo.toml)

Makefile

@@ -16,22 +16,19 @@ endif
 
 # Valid values are: amd64, arm64.
 TARGET_ARCH ?= $(NATIVE_ARCH)
-
-ifeq ($(NATIVE_ARCH),$(TARGET_ARCH))
-ARCH_PREFIX :=
-else ifeq ($(TARGET_ARCH),arm64)
-ARCH_PREFIX := aarch64-linux-gnu-
+ifeq ($(TARGET_ARCH),arm64)
+ARCH_PREFIX := aarch64
 else ifeq ($(TARGET_ARCH),amd64)
-ARCH_PREFIX := x86_64-linux-gnu-
+ARCH_PREFIX := x86_64
 else
 $(error Unsupported architecture: $(TARGET_ARCH))
 endif
 
 export TARGET_ARCH
 export CGO_ENABLED = 1
 export GOARCH = $(TARGET_ARCH)
-export CC = $(ARCH_PREFIX)gcc
-export OBJCOPY = $(ARCH_PREFIX)objcopy
+export CC = $(ARCH_PREFIX)-linux-gnu-gcc
+export OBJCOPY = $(ARCH_PREFIX)-linux-gnu-objcopy
 
 BRANCH = $(shell git rev-parse --abbrev-ref HEAD | tr -d '-' | tr '[:upper:]' '[:lower:]')
 COMMIT_SHORT_SHA = $(shell git rev-parse --short=8 HEAD)
@@ -78,18 +75,10 @@ ebpf-profiler: generate ebpf rust-components
 	go build $(GO_FLAGS) -tags $(GO_TAGS)
 
 rust-targets:
-ifeq ($(TARGET_ARCH),arm64)
-	rustup target add aarch64-unknown-linux-musl
-else ifeq ($(TARGET_ARCH),amd64)
-	rustup target add x86_64-unknown-linux-musl
-endif
+	rustup target add $(ARCH_PREFIX)-unknown-linux-musl
 
 rust-components: rust-targets
-ifeq ($(TARGET_ARCH),arm64)
-	RUSTFLAGS="--remap-path-prefix $(PWD)=/" cargo build --lib --release --target aarch64-unknown-linux-musl
-else ifeq ($(TARGET_ARCH),amd64)
-	RUSTFLAGS="--remap-path-prefix $(PWD)=/" cargo build --lib --release --target x86_64-unknown-linux-musl
-endif
+	RUSTFLAGS="--remap-path-prefix $(PWD)=/" cargo build --lib --release --target $(ARCH_PREFIX)-unknown-linux-musl
 
 rust-tests: rust-targets
 	cargo test
@@ -114,7 +103,7 @@ vanity-import-fix: $(PORTO)
 	@go install github.com/jcchavezs/porto/cmd/porto@latest
 	@porto --include-internal -w .
 
-test: generate ebpf test-deps
+test: generate ebpf test-deps rust-components
 	go test $(GO_FLAGS) -tags $(GO_TAGS) ./...
 
 TESTDATA_DIRS:= \
@@ -129,7 +118,7 @@ test-deps:
 
 TEST_INTEGRATION_BINARY_DIRS := tracer processmanager/ebpf support
 
-integration-test-binaries: generate ebpf
+integration-test-binaries: generate ebpf rust-components
 	$(foreach test_name, $(TEST_INTEGRATION_BINARY_DIRS), \
 		(go test -ldflags='-extldflags=-static' -trimpath -c \
 			-tags $(GO_TAGS),static_build,integration \

README.md

@@ -50,6 +50,10 @@ The resulting binary will be named <ebpf-profiler> in the current directory.
 ## Other OSes
 Since the profiler is Linux-only, macOS and Windows users need to set up a Linux VM to build and run the agent. Ensure the appropriate architecture is specified if using cross-compilation. Use the same make targets as above after the Linux environment is configured in the VM.
 
+## Supported Linux kernel version
+
+[7ddc23ea](https://github.com/open-telemetry/opentelemetry-ebpf-profiler/commit/7ddc23ea135a2e00fffc17850ab90534e9b63108) is the last commit with support for 4.19. Changes after this commit may require a minimal Linux kernel version of 5.4.
+
 ## Alternative Build (Without Docker)
 You can build the agent without Docker by directly installing the dependencies listed in the Dockerfile. Once dependencies are set up, simply run:
 ```sh
@@ -352,9 +356,10 @@ traces user-land will simply read and then clear this map on a timer.
 The BPF components are responsible for notifying user-land about new and exiting
 processes. An event about a new process is produced when we first interrupt it
 with the unwinders. Events about exiting processes are created with a
-`sched_process_exit` probe. In both cases the BPF code sends a perf event to
+`sched_process_free` tracepoint. In both cases the BPF code sends a perf event to
 notify user-land. We also re-report a PID if we detect execution in previously
-unknown memory region to prompt re-scan of the mappings.
+unknown memory region to prompt re-scan of the mappings. Finally, the profiler
+can also profile processes whose main thread exits, leaving other threads running.
 
 ### Network protocol
 

asm/amd/insn.go

@@ -0,0 +1,20 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package amd // import "go.opentelemetry.io/ebpf-profiler/asm/amd"
+import "bytes"
+
+// https://www.felixcloutier.com/x86/endbr64
+var opcodeEndBr64 = []byte{0xf3, 0x0f, 0x1e, 0xfa}
+
+// DecodeSkippable decodes an instruction that we don't care much about and are going to skip,
+// as golang.org/x/arch/x86/x86asm fails to decode it.
+// The second returned argument is the size of the decoded instruction to skip.
+func DecodeSkippable(code []byte) (ok bool, size int) {
+	switch {
+	case bytes.HasPrefix(code, opcodeEndBr64):
+		return true, len(opcodeEndBr64)
+	default:
+		return false, 0
+	}
+}

asm/amd/insn_test.go

@@ -0,0 +1,19 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package amd
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestEndBr64(t *testing.T) {
+	res, n := DecodeSkippable([]byte{0xF3, 0x0F, 0x1E, 0xFA})
+	assert.True(t, res)
+	assert.Equal(t, 4, n)
+
+	res, _ = DecodeSkippable([]byte{})
+	assert.False(t, res)
+}

asm/amd/regs_state.go

@@ -0,0 +1,66 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package amd // import "go.opentelemetry.io/ebpf-profiler/asm/amd"
+
+import "golang.org/x/arch/x86/x86asm"
+
+// regIndex returns index into RegsState.regs
+func regIndex(reg x86asm.Reg) int {
+	switch reg {
+	case x86asm.RAX, x86asm.EAX:
+		return 1
+	case x86asm.RBX, x86asm.EBX:
+		return 2
+	case x86asm.RCX, x86asm.ECX:
+		return 3
+	case x86asm.RDX, x86asm.EDX:
+		return 4
+	case x86asm.RDI, x86asm.EDI:
+		return 5
+	case x86asm.RSI, x86asm.ESI:
+		return 6
+	case x86asm.RBP, x86asm.EBP:
+		return 7
+	case x86asm.R8, x86asm.R8L:
+		return 8
+	case x86asm.R9, x86asm.R9L:
+		return 9
+	case x86asm.R10, x86asm.R10L:
+		return 10
+	case x86asm.R11, x86asm.R11L:
+		return 11
+	case x86asm.R12, x86asm.R12L:
+		return 12
+	case x86asm.R13, x86asm.R13L:
+		return 13
+	case x86asm.R14, x86asm.R14L:
+		return 14
+	case x86asm.R15, x86asm.R15L:
+		return 15
+	case x86asm.RSP, x86asm.ESP:
+		return 16
+	case x86asm.RIP:
+		return 17
+	default:
+		return 0
+	}
+}
+
+type RegsState struct {
+	regs [18]regState
+}
+
+func (r *RegsState) Set(reg x86asm.Reg, value, loadedFrom uint64) {
+	r.regs[regIndex(reg)].Value = value
+	r.regs[regIndex(reg)].LoadedFrom = loadedFrom
+}
+
+func (r *RegsState) Get(reg x86asm.Reg) (value, loadedFrom uint64) {
+	return r.regs[regIndex(reg)].Value, r.regs[regIndex(reg)].LoadedFrom
+}
+
+type regState struct {
+	LoadedFrom uint64
+	Value      uint64
+}

design-docs/00002-custom-labels/README.md

@@ -0,0 +1,40 @@
+Custom Labels
+=============
+
+# Meta
+
+- **Author(s)**: Tommy Reilly
+- **Start Date**: 2025-05-15
+- **Goal End Date**: 2025-06-15
+- **Primary Reviewers**: Florian Lehner, Timo Teräs, Brennan Vincent
+
+# Problem
+
+Sometimes understanding performance issues is hard because there's no way to dissect hotspots by attributes that aren't visible in the program structure. For instance in a database that uses a generic query execution path to execute all queries you may want to see how much CPU cycles are on behalf of internal queries vs external queries, or you might want to see which user is doing the most queries. This requires attaching metadata to each sample. In Go this is typically done with pprof labels and pprof data can be split out by different values of these labels (example: https://www.polarsignals.com/blog/posts/2021/04/13/demystifying-pprof-labels-with-go).
+
+In addition to pprof labels more examples of where custom labels could be used:
+
+- Trace IDs for supporting queries of CPU resources used by a particular traceid
+- Runtime metadata like "goid" so that CPU resources associated by a particular Goroutine can be discerned
+- Arbitrary application/workload specific metadata like user, client or query
+
+This design doc describes how we can surface Go pprof labels in the OTel profiler and lays the groundwork for doing similar things for other languages.
+
+# Success criteria
+
+- Any native language unwinder should be able to add custom labels to each sample, ie it should not be Go specific even if Go is the initial target
+- Custom labels should have its own trace type for enable/disable purposes even though it is technically not an unwinder
+- When disabled custom labels has little to no impact on performance or memory usage of the profiler
+- Custom labels should be limited so that even if a program has thousands of eligible labels the number supported is reasonably small (mostly enforced by eBPF itself)
+- Custom labels should be short and have fixed memory overhead
+- The custom labels should be made available to the reporter backend but otherwise it should be left up to implementors what to do with them
+
+# Scope
+
+The initial proposal will only deal with Go pprof labels which are just string/string key/value pairs, more custom labels for Go or other languages may be added in the future. The initial proposal is to get up to 10 labels in best effort fashion, if any eBPF errors occur there may be fewer labels and there is no proposed mechanism for deciding which labels to grab. Even though the OTel proto allows arbitrary types for the value the initial implementation will be scoped to just strings.
+
+# Proposed Solution
+
+The solution we propose is to add support for 10 64 byte custom labels associated with each sample with 16 bytes for the label key and 48 bytes for the label value. These will be stored in the Trace struct with the stack frame information for each sample so each Trace will be 640 bytes larger than before.
+
+In Go 1.23 labels are stored in a map so its non-deterministic which labels are read from the program, in Go 1.24+ the labels are stored in a list sorted by their keys so it will be first come first serve which labels are extracted. If the labels key or value is larger than 16/48 bytes they will be truncated. No effort is made to validate the strings from a UTF8 perspective.

go.mod

@@ -76,7 +76,7 @@ require (
 	go.opentelemetry.io/otel/trace v1.35.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.27.0 // indirect
-	golang.org/x/net v0.37.0 // indirect
+	golang.org/x/net v0.38.0 // indirect
 	golang.org/x/text v0.23.0 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20250102185135-69823020774d // indirect
 	google.golang.org/protobuf v1.36.1 // indirect

go.sum

@@ -170,8 +170,8 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
-golang.org/x/net v0.37.0 h1:1zLorHbz+LYj7MQlSf1+2tPIIgibq2eL5xkrGk6f+2c=
-golang.org/x/net v0.37.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8=
+golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8=
+golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=

internal/controller/controller.go

@@ -137,8 +137,8 @@ func (c *Controller) Start(ctx context.Context) error {
 		return fmt.Errorf("failed to attach scheduler monitor: %w", err)
 	}
 
-	// This log line is used in our system tests to verify if that the agent has started. So if you
-	// change this log line update also the system test.
+	// This log line is used in our system tests to verify if that the agent has started.
+	// So if you change this log line update also the system test.
 	log.Printf("Attached sched monitor")
 
 	if err := startTraceHandling(ctx, c.reporter, intervals, trc,