Crash with message - offset not found: net/http.Request:Method (1.21.5)

[lazyboson]

Describe the bug

I have written simple API in go -

package main

import (
	"math/rand"
	"net/http"
	"time"

	"github.com/labstack/echo/v4"
)

func main() {
	rand.Seed(time.Now().UnixNano())

	e := echo.New()

	// API endpoint to return a random number
	e.GET("/random-number", func(c echo.Context) error {
		randomNumber := rand.Intn(100)
		return c.JSON(http.StatusOK, map[string]int{"randomNumber": randomNumber})
	})

	// API endpoint to return a random dog breed
	e.GET("/random-dog-breed", func(c echo.Context) error {
		dogBreeds := []string{"Labrador", "German Shepherd", "Golden Retriever", "Bulldog", "Poodle", "Husky"}
		randomIndex := rand.Intn(len(dogBreeds))
		randomBreed := dogBreeds[randomIndex]
		return c.JSON(http.StatusOK, map[string]string{"randomDogBreed": randomBreed})
	})

	e.Logger.Fatal(e.Start(":8088"))
}

which Docker file -

# Build stage
FROM golang:latest AS build

WORKDIR /app

COPY . .

# Download and install any required third-party dependencies
RUN go mod download

# Build the application with optimizations
RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o main .

# Final stage
FROM alpine:latest

WORKDIR /app

# Copy only the necessary files from the build stage
COPY --from=build /app/main .

# Expose port 8080 to the outside world
EXPOSE 8088

# Set the entry point and command
ENTRYPOINT [ "./main" ]

Using above docker file created image and pushed on dockerhub. Then I am deploying this images in minikube cluster using below deployment file.

apiVersion: apps/v1
kind: Deployment
metadata:
  name: sapi-deployment
spec:
  replicas: 1
  selector:
    matchLabels:
      app: go-echo
  template:
    metadata:
      labels:
        app: go-echo
      annotations:
        sidecar.opentelemetry.io/inject: "true" 
        instrumentation.opentelemetry.io/inject-go: "true"
        instrumentation.opentelemetry.io/otel-go-auto-target-exe: "/app/main"
        instrumentation.opentelemetry.io/container-names: "simpleapi"

    spec:
      containers:
      - name: simpleapi
        image: lazyboson/simpleapi:latest
        env:
        - name: OTEL_GO_AUTO_TARGET_EXE
          value: "/app/main"
        - name: OTEL_EXPORTER_OTLP_ENDPOINT
          value: "http://192.168.49.2:4318"
        - name: OTEL_SERVICE_NAME
          value: "go-echo-service"
        securityContext:
          privileged: true
          runAsUser: 0
        ports:
        - containerPort: 8088
        
---
apiVersion: v1
kind: Service
metadata:
  name: go-echo-service
spec:
  selector:
    app: go-echo
  ports:
    - protocol: TCP
      port: 80
      targetPort: 8088
  type: LoadBalancer

otel-operator and collector is deployed with helm chat, is running perfectly fine. How ever i am gettign follwoing error in instrumentation.

"level":"info","ts":1702920148.2658122,"logger":"go.opentelemetry.io/auto","caller":"cli/main.go:91","msg":"building OpenTelemetry Go instrumentation ..."}
{"level":"info","ts":1702920150.340784,"logger":"Instrumentation.Analyzer","caller":"process/discover.go:69","msg":"found process","pid":8}
{"level":"info","ts":1702920150.3524644,"logger":"Instrumentation.Analyzer","caller":"process/analyze.go:98","msg":"found function","function_name":{"Name":"net/http.HandlerFunc.ServeHTTP","Offset":2171744,"ReturnOffsets":[2171790]}}
{"level":"info","ts":1702920150.3542163,"logger":"Instrumentation.Allocate.TracedProgram","caller":"ptrace/ptrace_linux.go:133","msg":"attach successfully","tid":14}
{"level":"info","ts":1702920150.3551383,"logger":"Instrumentation.Allocate.TracedProgram","caller":"ptrace/ptrace_linux.go:133","msg":"attach successfully","tid":15}
{"level":"info","ts":1702920150.3552485,"logger":"Instrumentation.Allocate.TracedProgram","caller":"ptrace/ptrace_linux.go:133","msg":"attach successfully","tid":16}
{"level":"info","ts":1702920150.3554611,"logger":"Instrumentation.Allocate.TracedProgram","caller":"ptrace/ptrace_linux.go:133","msg":"attach successfully","tid":17}
{"level":"info","ts":1702920150.3555498,"logger":"Instrumentation.Allocate.TracedProgram","caller":"ptrace/ptrace_linux.go:133","msg":"attach successfully","tid":8}
{"level":"info","ts":1702920150.3558319,"logger":"Instrumentation.Allocate.TracedProgram","caller":"ptrace/ptrace_linux.go:234","msg":"Detected linux kernel version","version":"5.15.0"}
{"level":"info","ts":1702920150.3561285,"logger":"Instrumentation.Allocate.TracedProgram","caller":"ptrace/ptrace_linux.go:246","msg":"mlock ret","ret":0}
{"level":"info","ts":1702920150.3561594,"logger":"Instrumentation.Allocate","caller":"process/allocate.go:64","msg":"Detaching from process","pid":8}
{"level":"info","ts":1702920150.356199,"logger":"Instrumentation.Allocate","caller":"process/allocate.go:43","msg":"mmaped remote memory","start_addr":"7FB792EE4000","end_addr":"7FB792F24000"}
{"level":"info","ts":1702920150.3562129,"logger":"Instrumentation","caller":"app/instrumentation.go:132","msg":"target process analysis completed","pid":8,"go_version":"1.21.5","dependencies":{"-buildmode=exe":null,"-compiler=gc":null,"CGO_ENABLED=0":null,"DefaultGODEBUG=panicnil=1":null,"GOAMD64=v1":null,"GOARCH=amd64":null,"GOOS=linux":null,"github.com/labstack/echo/v4":"4.11.3","github.com/labstack/gommon":"0.4.0","github.com/mattn/go-colorable":"0.1.13","github.com/mattn/go-isatty":"0.0.19","github.com/valyala/bytebufferpool":"1.0.0","github.com/valyala/fasttemplate":"1.2.2","golang.org/x/crypto":"0.14.0","golang.org/x/net":"0.17.0","golang.org/x/sys":"0.13.0","golang.org/x/text":"0.13.0","simpleapi":null,"std":"1.21.5"},"total_functions_found":1}
{"level":"info","ts":1702920150.3563123,"logger":"go.opentelemetry.io/auto","caller":"cli/main.go:98","msg":"starting instrumentation..."}
{"level":"info","ts":1702920150.3564134,"logger":"Instrumentation.Manager","caller":"instrumentation/manager.go:178","msg":"Mounting bpffs","allocations_details":{"StartAddr":140426420830208,"EndAddr":140426421092352}}
{"level":"info","ts":1702920150.3589058,"logger":"Instrumentation.Manager","caller":"instrumentation/manager.go:163","msg":"loading probes","name":"net/http"}
{"level":"error","ts":1702920150.3642852,"logger":"Instrumentation.Manager","caller":"instrumentation/manager.go:166","msg":"error while loading probes, cleaning up","name":"net/http","error":"offset not found: net/http.Request:Method (1.21.5)\noffset not found: net/http.Request:URL (1.21.5)\noffset not found: net/http.Request:ctx (1.21.5)\noffset not found: net/url.URL:Path (1.21.5)\noffset not found: net/http.Request:Header (1.21.5)\noffset not found: net/http.response:req (1.21.5)\noffset not found: net/http.response:status (1.21.5)\noffset not found: runtime.hmap:buckets (1.21.5)","stacktrace":"go.opentelemetry.io/auto/internal/pkg/instrumentation.(*Manager).load\n\t/app/internal/pkg/instrumentation/manager.go:166\ngo.opentelemetry.io/auto/internal/pkg/instrumentation.(*Manager).Run\n\t/app/internal/pkg/instrumentation/manager.go:121\ngo.opentelemetry.io/auto.(*Instrumentation).Run\n\t/app/instrumentation.go:150\nmain.main\n\t/app/cli/main.go:99\nruntime.main\n\t/usr/local/go/src/runtime/proc.go:267"}
{"level":"info","ts":1702920150.3658164,"logger":"Instrumentation.Manager","caller":"instrumentation/manager.go:191","msg":"Cleaning bpffs"}
{"level":"error","ts":1702920150.3658662,"logger":"go.opentelemetry.io/auto","caller":"cli/main.go:100","msg":"instrumentation crashed","error":"offset not found: net/http.Request:Method (1.21.5)\noffset not found: net/http.Request:URL (1.21.5)\noffset not found: net/http.Request:ctx (1.21.5)\noffset not found: net/url.URL:Path (1.21.5)\noffset not found: net/http.Request:Header (1.21.5)\noffset not found: net/http.response:req (1.21.5)\noffset not found: net/http.response:status (1.21.5)\noffset not found: runtime.hmap:buckets (1.21.5)","stacktrace":"main.main\n\t/app/cli/main.go:100\nruntime.main\n\t/usr/local/go/src/runtime/proc.go:267"}

Environment

• OS: [Ubuntu 22]
• Go Version: [1.20]

[vitorhugoro1]

Same error here when I try out with Docker, but sometimes return this another error

[RonFed]

@lazyboson The error you're getting should be resolved with the latest image of the go-auto-instrumentation Release v0.9.0-alpha
@vitorhugoro1 The error message regarding bpf_probe_write_user is a separate issue described in #290