diff --git a/host/host.go b/host/host.go index 1adc80f07..fa0f63bf9 100644 --- a/host/host.go +++ b/host/host.go @@ -9,7 +9,6 @@ import ( "fmt" "go.opentelemetry.io/ebpf-profiler/libpf" - "go.opentelemetry.io/ebpf-profiler/times" ) // FileID is used for unique identifiers for files, and is required to be 64-bits @@ -33,29 +32,3 @@ func (fid FileID) StringNoQuotes() string { func FileIDFromLibpf(id libpf.FileID) FileID { return FileID(id.Hi()) } - -type Frame struct { - File FileID - Lineno libpf.AddressOrLineno - Type libpf.FrameType - ReturnAddress bool -} - -type Trace struct { - Comm libpf.String - ProcessName libpf.String - ExecutablePath libpf.String - ContainerID libpf.String - Frames []Frame - KTime times.KTime - PID libpf.PID - TID libpf.PID - Origin libpf.Origin - OffTime int64 // Time a task was off-cpu in nanoseconds. - APMTraceID libpf.APMTraceID - APMTransactionID libpf.APMTransactionID - CPU int - EnvVars map[libpf.String]libpf.String - CustomLabels map[libpf.String]libpf.String - KernelFrames libpf.Frames -} diff --git a/internal/controller/controller.go b/internal/controller/controller.go index 69cfcd889..8ae8f8c3d 100644 --- a/internal/controller/controller.go +++ b/internal/controller/controller.go @@ -9,7 +9,6 @@ import ( "go.opentelemetry.io/ebpf-profiler/internal/log" - "go.opentelemetry.io/ebpf-profiler/host" "go.opentelemetry.io/ebpf-profiler/libpf" "go.opentelemetry.io/ebpf-profiler/metrics" "go.opentelemetry.io/ebpf-profiler/reporter" @@ -163,7 +162,7 @@ func (c *Controller) Shutdown() { func startTraceHandling(ctx context.Context, trc *tracer.Tracer) error { // Spawn monitors for the various result maps - traceCh := make(chan *host.Trace) + traceCh := make(chan *libpf.EbpfTrace) if err := trc.StartMapMonitors(ctx, traceCh); err != nil { return fmt.Errorf("failed to start map monitors: %v", err) diff --git a/interpreter/apmint/apmint.go b/interpreter/apmint/apmint.go index ab53f7373..283a90348 100644 --- a/interpreter/apmint/apmint.go +++ b/interpreter/apmint/apmint.go @@ -16,7 +16,6 @@ import ( "go.opentelemetry.io/ebpf-profiler/internal/log" - "go.opentelemetry.io/ebpf-profiler/host" "go.opentelemetry.io/ebpf-profiler/interpreter" "go.opentelemetry.io/ebpf-profiler/libpf" "go.opentelemetry.io/ebpf-profiler/libpf/pfelf" @@ -162,7 +161,7 @@ func (i *Instance) Detach(ebpf interpreter.EbpfHandler, pid libpf.PID) error { // NotifyAPMAgent sends out collected traces to the connected APM agent. func (i *Instance) NotifyAPMAgent( - pid libpf.PID, rawTrace *host.Trace, umTraceHash libpf.TraceHash, count uint16, + pid libpf.PID, rawTrace *libpf.EbpfTrace, umTraceHash libpf.TraceHash, count uint16, ) { if rawTrace.APMTransactionID == libpf.InvalidAPMSpanID || i.socket == nil { return diff --git a/interpreter/beam/beam.go b/interpreter/beam/beam.go index 91fe04d92..65fd6fb42 100644 --- a/interpreter/beam/beam.go +++ b/interpreter/beam/beam.go @@ -15,7 +15,6 @@ import ( "go.opentelemetry.io/ebpf-profiler/internal/log" - "go.opentelemetry.io/ebpf-profiler/host" "go.opentelemetry.io/ebpf-profiler/interpreter" "go.opentelemetry.io/ebpf-profiler/libpf" "go.opentelemetry.io/ebpf-profiler/lpm" @@ -227,14 +226,14 @@ func (i *beamInstance) Detach(interpreter.EbpfHandler, libpf.PID) error { return nil } -func (i *beamInstance) Symbolize(frame *host.Frame, frames *libpf.Frames) error { - if !frame.Type.IsInterpType(libpf.BEAM) { +func (r *beamInstance) Symbolize(ef libpf.EbpfFrame, frames *libpf.Frames) error { + if !ef.Type().IsInterpType(libpf.BEAM) { return interpreter.ErrMismatchInterpreterType } frames.Append(&libpf.Frame{ Type: libpf.BEAMFrame, - AddressOrLineno: frame.Lineno, + AddressOrLineno: libpf.AddressOrLineno(ef.Data()), }) return nil diff --git a/interpreter/dotnet/instance.go b/interpreter/dotnet/instance.go index ef3e7721f..ea8f43309 100644 --- a/interpreter/dotnet/instance.go +++ b/interpreter/dotnet/instance.go @@ -714,20 +714,20 @@ func (i *dotnetInstance) GetAndResetMetrics() ([]metrics.Metric, error) { }, nil } -func (i *dotnetInstance) Symbolize(frame *host.Frame, frames *libpf.Frames) error { - if !frame.Type.IsInterpType(libpf.Dotnet) { +func (i *dotnetInstance) Symbolize(ef libpf.EbpfFrame, frames *libpf.Frames) error { + if !ef.Type().IsInterpType(libpf.Dotnet) { return interpreter.ErrMismatchInterpreterType } sfCounter := successfailurecounter.New(&i.successCount, &i.failCount) defer sfCounter.DefaultToFailure() - codeHeaderAndType := frame.File - frameType := uint(codeHeaderAndType & 0x1f) + codeHeaderAndType := ef.Variable(0) + subframeType := uint(codeHeaderAndType & 0x1f) codeHeaderPtr := libpf.Address(codeHeaderAndType >> 5) - pcOffset := uint32(frame.Lineno) + pcOffset := uint32(ef.Data()) - switch frameType { + switch subframeType { case codeReadyToRun: // Ready to Run (Non-JIT) frame running directly code from a PE file module, err := i.getPEInfoByAddress(uint64(codeHeaderPtr)) @@ -756,7 +756,7 @@ func (i *dotnetInstance) Symbolize(frame *host.Frame, frames *libpf.Frames) erro break } - ilOffset := method.mapPCOffsetToILOffset(pcOffset, frame.ReturnAddress) + ilOffset := method.mapPCOffsetToILOffset(pcOffset, ef.Flags().ReturnAddress()) // The Line ID format is: // 4 bits Set to 0xf to indicate JIT frame. @@ -776,7 +776,7 @@ func (i *dotnetInstance) Symbolize(frame *host.Frame, frames *libpf.Frames) erro }) default: // Stub code - i.appendStubFrame(frames, frameType) + i.appendStubFrame(frames, subframeType) } sfCounter.ReportSuccess() diff --git a/interpreter/go/go.go b/interpreter/go/go.go index 1356558b7..cf31bb0b3 100644 --- a/interpreter/go/go.go +++ b/interpreter/go/go.go @@ -7,7 +7,6 @@ import ( "fmt" "sync/atomic" - "go.opentelemetry.io/ebpf-profiler/host" "go.opentelemetry.io/ebpf-profiler/interpreter" "go.opentelemetry.io/ebpf-profiler/libpf" "go.opentelemetry.io/ebpf-profiler/metrics" @@ -102,22 +101,24 @@ func (g *goInstance) Detach(_ interpreter.EbpfHandler, _ libpf.PID) error { return nil } -func (g *goInstance) Symbolize(frame *host.Frame, frames *libpf.Frames) error { - if !frame.Type.IsInterpType(libpf.Native) { +func (g *goInstance) Symbolize(ef libpf.EbpfFrame, frames *libpf.Frames) error { + if !ef.Type().IsInterpType(libpf.Native) { return interpreter.ErrMismatchInterpreterType } + sfCounter := successfailurecounter.New(&g.successCount, &g.failCount) defer sfCounter.DefaultToFailure() - sourceFile, lineNo, fn := g.d.pclntab.Symbolize(uintptr(frame.Lineno)) + address := ef.Data() + sourceFile, lineNo, fn := g.d.pclntab.Symbolize(uintptr(address)) if fn == "" { - return fmt.Errorf("failed to symbolize 0x%x", frame.Lineno) + return fmt.Errorf("failed to symbolize 0x%x", address) } frames.Append(&libpf.Frame{ Type: libpf.GoFrame, //TODO: File: convert the frame.File (host.FileID) to libpf.FileID here - AddressOrLineno: frame.Lineno, + AddressOrLineno: libpf.AddressOrLineno(address), FunctionName: libpf.Intern(fn), SourceFile: libpf.Intern(sourceFile), SourceLine: libpf.SourceLineno(lineNo), diff --git a/interpreter/go/go_test.go b/interpreter/go/go_test.go index bf4f25d5b..c035cfc84 100644 --- a/interpreter/go/go_test.go +++ b/interpreter/go/go_test.go @@ -50,12 +50,10 @@ func BenchmarkGolang(b *testing.B) { } frames := libpf.Frames{} + ef := libpf.NewEbpfFrame(libpf.NativeFrame, 0, 1, uint64(pc)) + ef[1] = uint64(hostFileID) - if err := gI.Symbolize(&host.Frame{ - File: hostFileID, - Lineno: libpf.AddressOrLineno(pc), - Type: libpf.FrameType(libpf.Native), - }, &frames); err != nil { + if err := gI.Symbolize(ef, &frames); err != nil { b.Fatalf("Failed to symbolize 0x%x: %v", pc, err) } diff --git a/interpreter/golabels/integrationtests/golabels_integration_test.go b/interpreter/golabels/integrationtests/golabels_integration_test.go index c079947a7..dedead1af 100644 --- a/interpreter/golabels/integrationtests/golabels_integration_test.go +++ b/interpreter/golabels/integrationtests/golabels_integration_test.go @@ -19,7 +19,7 @@ import ( "time" "github.com/stretchr/testify/require" - "go.opentelemetry.io/ebpf-profiler/host" + "go.opentelemetry.io/ebpf-profiler/libpf" "go.opentelemetry.io/ebpf-profiler/metrics" "go.opentelemetry.io/ebpf-profiler/tracer" tracertypes "go.opentelemetry.io/ebpf-profiler/tracer/types" @@ -107,7 +107,7 @@ func Test_Golabels(t *testing.T) { require.NoError(t, trc.EnableProfiling()) require.NoError(t, trc.AttachSchedMonitor()) - traceCh := make(chan *host.Trace) + traceCh := make(chan *libpf.EbpfTrace) require.NoError(t, trc.StartMapMonitors(ctx, traceCh)) wg := sync.WaitGroup{} diff --git a/interpreter/hotspot/instance.go b/interpreter/hotspot/instance.go index 1e578ef7a..b6d872667 100644 --- a/interpreter/hotspot/instance.go +++ b/interpreter/hotspot/instance.go @@ -867,16 +867,16 @@ func (d *hotspotInstance) SynchronizeMappings(ebpf interpreter.EbpfHandler, // Symbolize interpreters Hotspot eBPF uwinder given data containing target // process address and translates it to decorated frames expanding any inlined // frames to multiple new frames. -func (d *hotspotInstance) Symbolize(frame *host.Frame, frames *libpf.Frames) error { - if !frame.Type.IsInterpType(libpf.HotSpot) { +func (d *hotspotInstance) Symbolize(ef libpf.EbpfFrame, frames *libpf.Frames) error { + if !ef.Type().IsInterpType(libpf.HotSpot) { return interpreter.ErrMismatchInterpreterType } // Extract the HotSpot frame bitfields from the file and line variables - ptr := libpf.Address(frame.File) - subtype := uint32(frame.Lineno>>60) & 0xf - ripOrBci := uint32(frame.Lineno>>32) & 0x0fffffff - ptrCheck := uint32(frame.Lineno) + ptr := libpf.Address(ef.Variable(0)) + subtype := uint32(ef.Variable(1)>>60) & 0xf + ripOrBci := uint32(ef.Variable(1)>>32) & 0x0fffffff + ptrCheck := uint32(ef.Variable(1)) var err error sfCounter := successfailurecounter.New(&d.successCount, &d.failCount) diff --git a/interpreter/instancestubs.go b/interpreter/instancestubs.go index 75af5fb35..1eb82637b 100644 --- a/interpreter/instancestubs.go +++ b/interpreter/instancestubs.go @@ -4,7 +4,6 @@ package interpreter // import "go.opentelemetry.io/ebpf-profiler/interpreter" import ( - "go.opentelemetry.io/ebpf-profiler/host" "go.opentelemetry.io/ebpf-profiler/libc" "go.opentelemetry.io/ebpf-profiler/libpf" "go.opentelemetry.io/ebpf-profiler/metrics" @@ -26,7 +25,7 @@ func (is *InstanceStubs) UpdateLibcInfo(EbpfHandler, libpf.PID, libc.LibcInfo) e return nil } -func (is *InstanceStubs) Symbolize(*host.Frame, *libpf.Frames) error { +func (is *InstanceStubs) Symbolize(libpf.EbpfFrame, *libpf.Frames) error { return ErrMismatchInterpreterType } diff --git a/interpreter/multi.go b/interpreter/multi.go index b6372fc1e..fd74ab495 100644 --- a/interpreter/multi.go +++ b/interpreter/multi.go @@ -6,7 +6,6 @@ package interpreter // import "go.opentelemetry.io/ebpf-profiler/interpreter" import ( "errors" - "go.opentelemetry.io/ebpf-profiler/host" "go.opentelemetry.io/ebpf-profiler/internal/log" "go.opentelemetry.io/ebpf-profiler/libc" "go.opentelemetry.io/ebpf-profiler/libpf" @@ -116,10 +115,10 @@ func (m *MultiInstance) UpdateLibcInfo(ebpf EbpfHandler, pid libpf.PID, info lib } // Symbolize tries to symbolize the frame with each interpreter instance until one succeeds. -func (m *MultiInstance) Symbolize(ebpfFrame *host.Frame, frames *libpf.Frames) error { +func (m *MultiInstance) Symbolize(ef libpf.EbpfFrame, frames *libpf.Frames) error { // Try each interpreter in order for _, instance := range m.instances { - err := instance.Symbolize(ebpfFrame, frames) + err := instance.Symbolize(ef, frames) if err != ErrMismatchInterpreterType { return err } diff --git a/interpreter/nodev8/v8.go b/interpreter/nodev8/v8.go index fb5556b55..83946c1f1 100644 --- a/interpreter/nodev8/v8.go +++ b/interpreter/nodev8/v8.go @@ -168,7 +168,6 @@ import ( "github.com/elastic/go-freelru" - "go.opentelemetry.io/ebpf-profiler/host" "go.opentelemetry.io/ebpf-profiler/interpreter" "go.opentelemetry.io/ebpf-profiler/libpf" "go.opentelemetry.io/ebpf-profiler/libpf/pfelf" @@ -1701,21 +1700,21 @@ func (i *v8Instance) symbolizeCode(code *v8Code, delta uint64, returnAddress boo return nil } -func (i *v8Instance) Symbolize(frame *host.Frame, frames *libpf.Frames) error { - if !frame.Type.IsInterpType(libpf.V8) { +func (i *v8Instance) Symbolize(ef libpf.EbpfFrame, frames *libpf.Frames) error { + if !ef.Type().IsInterpType(libpf.V8) { return interpreter.ErrMismatchInterpreterType } sfCounter := successfailurecounter.New(&i.successCount, &i.failCount) defer sfCounter.DefaultToFailure() - pointerAndType := libpf.Address(frame.File) - deltaOrMarker := uint64(frame.Lineno) - frameType := pointerAndType & support.V8FileTypeMask + pointerAndType := libpf.Address(ef.Variable(0)) + deltaOrMarker := uint64(ef.Variable(1)) + subframeType := pointerAndType & support.V8FileTypeMask pointer := pointerAndType&^support.V8FileTypeMask | HeapObjectTag var err error - switch frameType { + switch subframeType { case support.V8FileTypeMarker: // This is a stub V8 frame, with deltaOrMarker containing the marker. // Convert the V8 build specific marker ID to a static ID and symbolize @@ -1726,16 +1725,16 @@ func (i *v8Instance) Symbolize(frame *host.Frame, frames *libpf.Frames) error { case support.V8FileTypeNativeCode, support.V8FileTypeNativeJSFunc: var code *v8Code codeCookie := uint32(deltaOrMarker & support.V8LineCookieMask >> support.V8LineCookieShift) - if frameType == support.V8FileTypeNativeCode { + if subframeType == support.V8FileTypeNativeCode { code, err = i.getCode(pointer, codeCookie) } else { code, err = i.getCodeFromJSFunc(pointer, codeCookie) } if err == nil { - err = i.symbolizeCode(code, deltaOrMarker, frame.ReturnAddress, frames) + err = i.symbolizeCode(code, deltaOrMarker, ef.Flags().ReturnAddress(), frames) } default: - err = fmt.Errorf("unsupported frame type %#x", frameType) + err = fmt.Errorf("unsupported frame type %#x", subframeType) } if err != nil { // TODO: emit error frame diff --git a/interpreter/perl/instance.go b/interpreter/perl/instance.go index 678237117..a42a26706 100644 --- a/interpreter/perl/instance.go +++ b/interpreter/perl/instance.go @@ -13,7 +13,6 @@ import ( "github.com/zeebo/xxh3" "go.opentelemetry.io/ebpf-profiler/internal/log" - "go.opentelemetry.io/ebpf-profiler/host" "go.opentelemetry.io/ebpf-profiler/interpreter" "go.opentelemetry.io/ebpf-profiler/libc" "go.opentelemetry.io/ebpf-profiler/libpf" @@ -385,8 +384,8 @@ func (i *perlInstance) getCOP(copAddr libpf.Address, funcName libpf.String) ( return c, nil } -func (i *perlInstance) Symbolize(frame *host.Frame, frames *libpf.Frames) error { - if !frame.Type.IsInterpType(libpf.Perl) { +func (i *perlInstance) Symbolize(ef libpf.EbpfFrame, frames *libpf.Frames) error { + if !ef.Type().IsInterpType(libpf.Perl) { return interpreter.ErrMismatchInterpreterType } @@ -394,13 +393,13 @@ func (i *perlInstance) Symbolize(frame *host.Frame, frames *libpf.Frames) error defer sfCounter.DefaultToFailure() functionName := interpreter.TopLevelFunctionName - if gvAddr := libpf.Address(frame.File); gvAddr != 0 { + if gvAddr := libpf.Address(ef.Variable(0)); gvAddr != 0 { var err error if functionName, err = i.getGV(gvAddr, false); err != nil { return fmt.Errorf("failed to get Perl GV %x: %v", gvAddr, err) } } - copAddr := libpf.Address(frame.Lineno) + copAddr := libpf.Address(ef.Variable(1)) cop, err := i.getCOP(copAddr, functionName) if err != nil { return fmt.Errorf("failed to get Perl COP %x: %v", copAddr, err) diff --git a/interpreter/php/instance.go b/interpreter/php/instance.go index 4b9f62dfe..d7121d3ba 100644 --- a/interpreter/php/instance.go +++ b/interpreter/php/instance.go @@ -12,7 +12,6 @@ import ( "github.com/elastic/go-freelru" - "go.opentelemetry.io/ebpf-profiler/host" "go.opentelemetry.io/ebpf-profiler/interpreter" "go.opentelemetry.io/ebpf-profiler/libpf" "go.opentelemetry.io/ebpf-profiler/metrics" @@ -168,22 +167,22 @@ func (i *phpInstance) getFunction(addr libpf.Address, typeInfo uint32) (*phpFunc return pf, nil } -func (i *phpInstance) Symbolize(frame *host.Frame, frames *libpf.Frames) error { +func (i *phpInstance) Symbolize(ef libpf.EbpfFrame, frames *libpf.Frames) error { // With Symbolize() in opcacheInstance there is a dedicated function to symbolize JITTed // PHP frames. But as we also attach phpInstance to PHP processes with JITTed frames, we // use this function to symbolize all PHP frames, as the process to do so is the same. - if !frame.Type.IsInterpType(libpf.PHP) && - !frame.Type.IsInterpType(libpf.PHPJIT) { + if !ef.Type().IsInterpType(libpf.PHP) && + !ef.Type().IsInterpType(libpf.PHPJIT) { return interpreter.ErrMismatchInterpreterType } sfCounter := successfailurecounter.New(&i.successCount, &i.failCount) defer sfCounter.DefaultToFailure() - funcPtr := libpf.Address(frame.File) + funcPtr := libpf.Address(ef.Variable(0)) // We pack type info and the line number into linenos - typeInfo := uint32(frame.Lineno >> 32) - line := frame.Lineno & 0xffffffff + typeInfo := uint32(ef.Variable(1) >> 32) + line := uint32(ef.Variable(1)) f, err := i.getFunction(funcPtr, typeInfo) if err != nil { @@ -191,8 +190,8 @@ func (i *phpInstance) Symbolize(frame *host.Frame, frames *libpf.Frames) error { } funcOff := uint32(0) - if f.lineStart != 0 && libpf.AddressOrLineno(f.lineStart) <= line { - funcOff = uint32(line) - f.lineStart + if f.lineStart != 0 && f.lineStart <= line { + funcOff = line - f.lineStart } frames.Append(&libpf.Frame{ diff --git a/interpreter/python/python.go b/interpreter/python/python.go index cc4320d60..0e0a4026d 100644 --- a/interpreter/python/python.go +++ b/interpreter/python/python.go @@ -24,7 +24,6 @@ import ( "github.com/elastic/go-freelru" - "go.opentelemetry.io/ebpf-profiler/host" "go.opentelemetry.io/ebpf-profiler/interpreter" "go.opentelemetry.io/ebpf-profiler/libc" "go.opentelemetry.io/ebpf-profiler/libpf" @@ -549,15 +548,15 @@ func (p *pythonInstance) getCodeObject(addr libpf.Address, return pco, nil } -func (p *pythonInstance) Symbolize(frame *host.Frame, frames *libpf.Frames) error { - if !frame.Type.IsInterpType(libpf.Python) { +func (p *pythonInstance) Symbolize(ef libpf.EbpfFrame, frames *libpf.Frames) error { + if !ef.Type().IsInterpType(libpf.Python) { return interpreter.ErrMismatchInterpreterType } // Extract the Python frame bitfields from the file and line variables - ptr := libpf.Address(frame.File) - lastI := uint32(frame.Lineno>>32) & 0x0fffffff - objectID := uint32(frame.Lineno) + ptr := libpf.Address(ef.Variable(0)) + lastI := uint32(ef.Variable(1)>>32) & 0x0fffffff + objectID := uint32(ef.Variable(1)) sfCounter := successfailurecounter.New(&p.successCount, &p.failCount) defer sfCounter.DefaultToFailure() diff --git a/interpreter/ruby/ruby.go b/interpreter/ruby/ruby.go index bfa05ec2e..193de08b1 100644 --- a/interpreter/ruby/ruby.go +++ b/interpreter/ruby/ruby.go @@ -20,7 +20,6 @@ import ( "github.com/elastic/go-freelru" - "go.opentelemetry.io/ebpf-profiler/host" "go.opentelemetry.io/ebpf-profiler/interpreter" "go.opentelemetry.io/ebpf-profiler/libpf" "go.opentelemetry.io/ebpf-profiler/libpf/pfelf" @@ -629,8 +628,8 @@ func (r *rubyInstance) getRubyLineNo(iseqBody libpf.Address, pc uint64) (uint32, return lineNo, nil } -func (r *rubyInstance) Symbolize(frame *host.Frame, frames *libpf.Frames) error { - if !frame.Type.IsInterpType(libpf.Ruby) { +func (r *rubyInstance) Symbolize(ef libpf.EbpfFrame, frames *libpf.Frames) error { + if !ef.Type().IsInterpType(libpf.Ruby) { return interpreter.ErrMismatchInterpreterType } vms := &r.r.vmStructs @@ -643,11 +642,11 @@ func (r *rubyInstance) Symbolize(frame *host.Frame, frames *libpf.Frames) error // // rb_iseq_constant_body // https://github.com/ruby/ruby/blob/5445e0435260b449decf2ac16f9d09bae3cafe72/vm_core.h#L311 - iseqBody := libpf.Address(frame.File) + iseqBody := libpf.Address(ef.Variable(0)) // The Ruby VM program counter that was extracted from the current call frame is embedded in // the Linenos field. - pc := frame.Lineno + pc := ef.Variable(1) lineNo, err := r.getRubyLineNo(iseqBody, uint64(pc)) if err != nil { diff --git a/interpreter/types.go b/interpreter/types.go index 2577fa1f0..1227e011d 100644 --- a/interpreter/types.go +++ b/interpreter/types.go @@ -149,7 +149,7 @@ type Instance interface { // Symbolize converts one ebpf frame to one or more (if inlining was expanded) libpf.Frame. // The resulting libpf.Frame values are appended to frames. - Symbolize(ebpfFrame *host.Frame, frames *libpf.Frames) error + Symbolize(ef libpf.EbpfFrame, frames *libpf.Frames) error // GetAndResetMetrics collects the metrics from the Instance and resets // the counters to their initial value. diff --git a/libpf/frameflags.go b/libpf/frameflags.go new file mode 100644 index 000000000..bf744a5b9 --- /dev/null +++ b/libpf/frameflags.go @@ -0,0 +1,23 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package libpf // import "go.opentelemetry.io/ebpf-profiler/libpf" + +import ( + "go.opentelemetry.io/ebpf-profiler/support" +) + +// FrameFlags defines the flags of an ebpf frame. +type FrameFlags uint8 + +func (ff FrameFlags) Error() bool { + return ff&support.FrameFlagError != 0 +} + +func (ff FrameFlags) ReturnAddress() bool { + return ff&support.FrameFlagReturnAddress != 0 +} + +func (ff FrameFlags) PIDSpecific() bool { + return ff&support.FrameFlagPidSpecific != 0 +} diff --git a/libpf/frametype.go b/libpf/frametype.go index 56817d70e..53337a22f 100644 --- a/libpf/frametype.go +++ b/libpf/frametype.go @@ -18,7 +18,7 @@ import ( // - A fatal failure that caused further unwinding to be aborted. This is indicated using the // special value support.FrameMarkerAbort (0xFF). It thus also contains the error bit, but // does not fit into the `InterpreterType` enum. -type FrameType int +type FrameType uint8 // Convenience shorthands to create various frame types. // @@ -26,9 +26,9 @@ type FrameType int // methods to query the required information (IsError, Interpreter, ...) to improve forward // compatibility and clarify intentions. const ( - // unknownFrame indicates a frame of an unknown interpreter. - // If this appears, it's likely a bug somewhere. - unknownFrame FrameType = support.FrameMarkerUnknown + // UnknownFrame indicates a frame of an unknown interpreter. + // Typically an error frame not associated with an interpreter. + UnknownFrame FrameType = support.FrameMarkerUnknown // PHPFrame identifies PHP interpreter frames. PHPFrame FrameType = support.FrameMarkerPHP // PHPJITFrame identifies PHP JIT interpreter frames. @@ -53,29 +53,26 @@ const ( GoFrame FrameType = support.FrameMarkerGo // BEAMFrame identifies the BEAM interpreter frames. BEAMFrame FrameType = support.FrameMarkerBEAM - // AbortFrame identifies frames that report that further unwinding was aborted due to an error. - AbortFrame FrameType = support.FrameMarkerAbort ) const ( abortFrameName = "abort-marker" + + errorBit = 0x80 + + abortFrame = errorBit | UnknownFrame ) func FrameTypeFromString(name string) FrameType { if name == abortFrameName { - return AbortFrame + return abortFrame } return InterpreterTypeFromString(name).Frame() } // Interpreter returns the interpreter that produced the frame. func (ty FrameType) Interpreter() InterpreterType { - switch ty { - case support.FrameMarkerAbort, support.FrameMarkerUnknown: - return UnknownInterp - default: - return InterpreterType(ty &^ support.FrameMarkerErrorBit) - } + return InterpreterType(ty &^ errorBit) } // IsInterpType checks whether the frame type belongs to the given interpreter. @@ -85,18 +82,23 @@ func (ty FrameType) IsInterpType(ity InterpreterType) bool { // Error adds the error bit into the frame type. func (ty FrameType) Error() FrameType { - return ty | support.FrameMarkerErrorBit + return ty | errorBit } // IsError checks whether the frame is an error frame. func (ty FrameType) IsError() bool { - return ty&support.FrameMarkerErrorBit != 0 + return ty&errorBit != 0 +} + +// IsAbort checks whether the frame is an abort frame. +func (ty FrameType) IsAbort() bool { + return ty == abortFrame } // String implements the Stringer interface. func (ty FrameType) String() string { switch ty { - case support.FrameMarkerAbort: + case abortFrame: return abortFrameName default: interp := ty.Interpreter() diff --git a/libpf/frametype_test.go b/libpf/frametype_test.go index 52dc0b93d..730e56aa6 100644 --- a/libpf/frametype_test.go +++ b/libpf/frametype_test.go @@ -12,8 +12,8 @@ import ( func TestFrameTypeFromString(t *testing.T) { // Simple check whether all FrameType values can be converted to string and back. for _, ft := range []FrameType{ - unknownFrame, PHPFrame, PythonFrame, NativeFrame, KernelFrame, HotSpotFrame, RubyFrame, - PerlFrame, V8Frame, DotnetFrame, AbortFrame} { + UnknownFrame, PHPFrame, PythonFrame, NativeFrame, KernelFrame, HotSpotFrame, RubyFrame, + PerlFrame, V8Frame, DotnetFrame} { t.Run(ft.String(), func(t *testing.T) { name := ft.String() result := FrameTypeFromString(name) diff --git a/libpf/interpretertype.go b/libpf/interpretertype.go index 55643b208..ff581e117 100644 --- a/libpf/interpretertype.go +++ b/libpf/interpretertype.go @@ -52,7 +52,7 @@ const ( // Frame converts the interpreter type into the corresponding frame type. func (i InterpreterType) Frame() FrameType { if i >= pseudoInterpreterStart { - return unknownFrame + return UnknownFrame } return FrameType(i) diff --git a/libpf/libpf_test.go b/libpf/libpf_test.go index 38dfea1e8..c59124d19 100644 --- a/libpf/libpf_test.go +++ b/libpf/libpf_test.go @@ -17,7 +17,7 @@ func TestTraceType(t *testing.T) { str string }{ { - ty: AbortFrame, + ty: abortFrame, isErr: true, interp: UnknownInterp, str: "abort-marker", diff --git a/libpf/trace.go b/libpf/trace.go index 44e03e8fb..09a7ac8cd 100644 --- a/libpf/trace.go +++ b/libpf/trace.go @@ -100,3 +100,60 @@ type Trace struct { Hash TraceHash CustomLabels map[String]String } + +// EbpfTrace represents a stack trace from Ebpf code. +type EbpfTrace struct { + Comm String + ProcessName String + ExecutablePath String + ContainerID String + KTime int64 + PID PID + TID PID + Origin Origin + OffTime int64 // Time a task was off-cpu in nanoseconds. + APMTraceID APMTraceID + APMTransactionID APMTransactionID + CPU int + NumFrames int + EnvVars map[String]String + CustomLabels map[String]String + KernelFrames Frames + FrameData []uint64 + FrameDataBuf [3072]uint64 +} + +type EbpfFrame []uint64 + +// The below code must match ebpf tracemgmt.h frame_header() layout. + +// NewEbpfFrame creates a new EbpfFrame slice with given header information. +// Typically used for testing only. +func NewEbpfFrame(ty FrameType, ff FrameFlags, l uint8, data uint64) []uint64 { + val := uint64(ty) << 60 + val |= uint64(ff) << 56 + val |= uint64(l) << 52 + ef := make([]uint64, l) + ef[0] = val | data + return ef +} + +func (f EbpfFrame) Type() FrameType { + return FrameType(f[0] >> 60) +} + +func (f EbpfFrame) Flags() FrameFlags { + return FrameFlags((f[0] >> 56) & 0xf) +} + +func (f EbpfFrame) Length() uint8 { + return uint8(f[0]>>52) & 0xf +} + +func (f EbpfFrame) Data() uint64 { + return uint64(f[0]) & 0xfffffffffffff +} + +func (f EbpfFrame) Variable(ndx int) uint64 { + return f[ndx+1] +} diff --git a/processmanager/manager.go b/processmanager/manager.go index 046fc7009..5d98f1188 100644 --- a/processmanager/manager.go +++ b/processmanager/manager.go @@ -8,6 +8,7 @@ import ( "context" "errors" "fmt" + "hash/fnv" "slices" "time" @@ -18,6 +19,7 @@ import ( "go.opentelemetry.io/ebpf-profiler/interpreter" "go.opentelemetry.io/ebpf-profiler/interpreter/apmint" "go.opentelemetry.io/ebpf-profiler/libpf" + "go.opentelemetry.io/ebpf-profiler/libpf/pfunsafe" "go.opentelemetry.io/ebpf-profiler/lpm" "go.opentelemetry.io/ebpf-profiler/metrics" "go.opentelemetry.io/ebpf-profiler/nativeunwind" @@ -178,7 +180,7 @@ func collectInterpreterMetrics(ctx context.Context, pm *ProcessManager, func (pm *ProcessManager) Close() { } -func (pm *ProcessManager) symbolizeFrame(pid libpf.PID, bpfFrame *host.Frame, frames *libpf.Frames) error { +func (pm *ProcessManager) symbolizeFrame(pid libpf.PID, data []uint64, frames *libpf.Frames) error { pm.mu.Lock() defer pm.mu.Unlock() @@ -187,7 +189,7 @@ func (pm *ProcessManager) symbolizeFrame(pid libpf.PID, bpfFrame *host.Frame, fr } for _, instance := range pm.interpreters[pid] { - if err := instance.Symbolize(bpfFrame, frames); err != nil { + if err := instance.Symbolize(data, frames); err != nil { if errors.Is(err, interpreter.ErrMismatchInterpreterType) { // The interpreter type of instance did not match the type of frame. // So continue with the next interpreter instance for this PID. @@ -204,15 +206,15 @@ func (pm *ProcessManager) symbolizeFrame(pid libpf.PID, bpfFrame *host.Frame, fr // convertFrame converts one host Frame to one or more libpf.Frames. It returns true // if non-trivial cacheable conversion was done. -func (pm *ProcessManager) convertFrame(pid libpf.PID, frame *host.Frame, dst *libpf.Frames) bool { - switch frame.Type.Interpreter() { +func (pm *ProcessManager) convertFrame(pid libpf.PID, ef libpf.EbpfFrame, dst *libpf.Frames) bool { + switch ef.Type().Interpreter() { case libpf.UnknownInterp, libpf.Kernel: log.Errorf("Unexpected frame type 0x%02X (neither error nor usermode frame)", - uint8(frame.Type)) + uint8(ef.Type())) case libpf.Native: // Attempt symbolization of native frames. It is best effort and // provides non-symbolized frames if no native symbolizer is active. - if err := pm.symbolizeFrame(pid, frame, dst); err == nil { + if err := pm.symbolizeFrame(pid, ef, dst); err == nil { return true } @@ -230,33 +232,34 @@ func (pm *ProcessManager) convertFrame(pid libpf.PID, frame *host.Frame, dst *li // Optimally we'd subtract the size of the call instruction here instead // of doing `- 1`, but disassembling backwards is quite difficult for // variable length instruction sets like X86. - relativeRIP := frame.Lineno - if frame.ReturnAddress { + fileID := host.FileID(ef.Variable(0)) + lineno := libpf.Address(ef.Data()) + relativeRIP := lineno + if ef.Flags().ReturnAddress() { relativeRIP-- } // Locate mapping info for the frame. - mapping := pm.findMappingForTrace(pid, frame.File, - libpf.Address(frame.Lineno)) + mapping := pm.findMappingForTrace(pid, fileID, lineno) dst.Append(&libpf.Frame{ - Type: frame.Type, - AddressOrLineno: relativeRIP, + Type: ef.Type(), + AddressOrLineno: libpf.AddressOrLineno(relativeRIP), Mapping: mapping, }) default: - err := pm.symbolizeFrame(pid, frame, dst) + err := pm.symbolizeFrame(pid, ef, dst) if err == nil { return true } log.Debugf("symbolization failed for PID %d, frame type %d: %v", - pid, frame.Type, err) - dst.Append(&libpf.Frame{Type: frame.Type}) + pid, ef.Type(), err) + dst.Append(&libpf.Frame{Type: ef.Type()}) } return false } func (pm *ProcessManager) maybeNotifyAPMAgent( - rawTrace *host.Trace, umTraceHash libpf.TraceHash, count uint16, + rawTrace *libpf.EbpfTrace, umTraceHash libpf.TraceHash, count uint16, ) string { pm.mu.RLock() // Keeping the lock until end of the function is needed because inner map can be modified @@ -286,16 +289,18 @@ func (pm *ProcessManager) maybeNotifyAPMAgent( } func hashFrameCacheKey(fk frameCacheKey) uint32 { - return uint32(uint64(fk.Frame.File) + uint64(fk.Frame.Lineno)) + h := fnv.New32a() + h.Write(pfunsafe.FromSlice(fk.data[:])) + return h.Sum32() } // HandleTrace processes and reports the given host.Trace. This function // is not re-entrant due to frameCache not being synced. If the tracer is // later updated to distribute trace handling to goroutine pool, the caching // strategy needs to be updated accordingly. -func (pm *ProcessManager) HandleTrace(bpfTrace *host.Trace) { +func (pm *ProcessManager) HandleTrace(bpfTrace *libpf.EbpfTrace) { meta := &samples.TraceEventMeta{ - Timestamp: libpf.UnixTime64(bpfTrace.KTime.UnixNano()), + Timestamp: libpf.UnixTime64(times.KTime(bpfTrace.KTime).UnixNano()), Comm: bpfTrace.Comm, PID: bpfTrace.PID, TID: bpfTrace.TID, @@ -312,7 +317,7 @@ func (pm *ProcessManager) HandleTrace(bpfTrace *host.Trace) { pid := bpfTrace.PID kernelFramesLen := len(bpfTrace.KernelFrames) trace := &libpf.Trace{ - Frames: make(libpf.Frames, kernelFramesLen, kernelFramesLen+len(bpfTrace.Frames)), + Frames: make(libpf.Frames, kernelFramesLen, kernelFramesLen+bpfTrace.NumFrames), CustomLabels: bpfTrace.CustomLabels, } copy(trace.Frames, bpfTrace.KernelFrames) @@ -320,28 +325,24 @@ func (pm *ProcessManager) HandleTrace(bpfTrace *host.Trace) { cacheMiss := uint64(0) cacheHit := uint64(0) - for i := range bpfTrace.Frames { - frame := &bpfTrace.Frames[i] - if frame.Type.IsError() { + for frames := libpf.EbpfFrame(bpfTrace.FrameData); len(frames) > 0; frames = frames[frames.Length():] { + frame := frames[:frames.Length()] + if frame.Flags().Error() { if !pm.filterErrorFrames { trace.Frames.Append(&libpf.Frame{ - Type: frame.Type, - AddressOrLineno: frame.Lineno, + Type: frame.Type().Error(), + AddressOrLineno: libpf.AddressOrLineno(frame.Data()), }) } continue } oldLen := len(trace.Frames) - key := frameCacheKey{Frame: *frame} - switch frame.Type { - case libpf.NativeFrame, libpf.KernelFrame: - // The native frames can be cached for all PIDs. - default: - // By default the per-interpreter frames have cached entry - // specific to the PID. - key.PID = pid + key := frameCacheKey{} + if frame.Flags().PIDSpecific() { + key.pid = pid } + copy(key.data[:], frame) if cached, ok := pm.frameCache.GetAndRefresh(key, frameCacheLifetime); ok { // Fast path cacheHit++ diff --git a/processmanager/types.go b/processmanager/types.go index 014f25081..ffe8e4ec5 100644 --- a/processmanager/types.go +++ b/processmanager/types.go @@ -9,7 +9,6 @@ import ( lru "github.com/elastic/go-freelru" - "go.opentelemetry.io/ebpf-profiler/host" "go.opentelemetry.io/ebpf-profiler/interpreter" "go.opentelemetry.io/ebpf-profiler/libc" "go.opentelemetry.io/ebpf-profiler/libpf" @@ -35,8 +34,10 @@ type elfInfo struct { // frameCacheKey is the LRU cache key for caching frames. type frameCacheKey struct { - host.Frame - libpf.PID + // pid is the PID of the process if the frame had FRAME_FLAG_PID_SPECIFIC set + pid libpf.PID + // data is the frame data: frame header and the two first variable fields + data [3]uint64 } // ProcessManager is responsible for managing the events happening throughout the lifespan of a diff --git a/support/ebpf/beam_tracer.ebpf.c b/support/ebpf/beam_tracer.ebpf.c index 3ddb69ac5..3abeb4cfd 100644 --- a/support/ebpf/beam_tracer.ebpf.c +++ b/support/ebpf/beam_tracer.ebpf.c @@ -40,6 +40,19 @@ typedef struct BEAMRangesSearchCache { BEAMRangeEntry first, mid, last; } BEAMRangesSearchCache; +static EBPF_INLINE ErrorCode push_beam(UnwindState *state, Trace *trace, u64 range_start) +{ + const u8 ra_flag = state->return_address ? FRAME_FLAG_RETURN_ADDRESS : 0; + + // `pc` is in the `current_range` CodeHeader + u64 *data = push_frame(state, trace, FRAME_MARKER_BEAM, ra_flag, state->pc, 1); + if (!data) { + return false; + } + data[0] = range_start; + return true; +} + static EBPF_INLINE ErrorCode unwind_one_beam_frame(PerCPURecord *record, BEAMProcInfo *info, BEAMRangesSearchCache *ranges) { @@ -64,9 +77,6 @@ unwind_one_beam_frame(PerCPURecord *record, BEAMProcInfo *info, BEAMRangesSearch } else if (pc >= current_range.end) { low = current + 1; } else { - // `pc` is in the `current_range` CodeHeader - _push_with_return_address( - trace, current_range.start, pc, FRAME_MARKER_BEAM, state->return_address); break; } @@ -84,6 +94,10 @@ unwind_one_beam_frame(PerCPURecord *record, BEAMProcInfo *info, BEAMRangesSearch return ERR_BEAM_RANGE_SEARCH_EXHAUSTED; } + if (!push_beam(state, trace, current_range.start)) { + return ERR_STACK_LENGTH_EXCEEDED; + } + if (info->frame_pointers_enabled) { if (!unwinder_unwind_frame_pointer(state)) { DEBUG_PRINT("beam: invalid frame pointer"); @@ -147,7 +161,7 @@ static EBPF_INLINE int unwind_beam(struct pt_regs *ctx) goto exit; } - DEBUG_PRINT("==== unwind_beam %d, pc: 0x%llx ====", trace->stack_len, state->pc); + DEBUG_PRINT("==== unwind_beam %d, pc: 0x%llx ====", trace->num_frames, state->pc); // "the_active_code_index" symbol is from: // https://github.com/erlang/otp/blob/OTP-27.2.4/erts/emulator/beam/code_ix.c#L46 @@ -227,4 +241,4 @@ static EBPF_INLINE int unwind_beam(struct pt_regs *ctx) return -1; } -MULTI_USE_FUNC(unwind_beam) \ No newline at end of file +MULTI_USE_FUNC(unwind_beam) diff --git a/support/ebpf/bpfdefs.h b/support/ebpf/bpfdefs.h index ede025598..70ab704d2 100644 --- a/support/ebpf/bpfdefs.h +++ b/support/ebpf/bpfdefs.h @@ -41,6 +41,7 @@ extern u32 with_debug_output; int bpf_tail_call(void *ctx, void *map, int index); unsigned long long bpf_ktime_get_ns(void); int bpf_get_current_comm(void *, int); +int bpf_perf_event_output(void *, void *, unsigned long long, void *, int); static inline long bpf_probe_read_user(void *buf, u32 sz, const void *ptr) { @@ -75,17 +76,6 @@ static inline int bpf_map_delete_elem(UNUSED void *map, UNUSED const void *key) return -1; } -static inline int bpf_perf_event_output( - UNUSED void *ctx, - UNUSED void *map, - UNUSED unsigned long long flags, - UNUSED void *data, - UNUSED int size) -{ - - return 0; -} - static inline int bpf_get_stackid(UNUSED void *ctx, UNUSED void *map, UNUSED u64 flags) { return -1; diff --git a/support/ebpf/dotnet_tracer.ebpf.c b/support/ebpf/dotnet_tracer.ebpf.c index 69b1587e4..2c26535ec 100644 --- a/support/ebpf/dotnet_tracer.ebpf.c +++ b/support/ebpf/dotnet_tracer.ebpf.c @@ -169,11 +169,18 @@ static EBPF_INLINE ErrorCode dotnet_find_code_start(PerCPURecord *record, u64 pc } // Record a Dotnet frame -static EBPF_INLINE ErrorCode -push_dotnet(Trace *trace, u64 code_header_ptr, u64 pc_offset, bool return_address) +static EBPF_INLINE ErrorCode push_dotnet( + UnwindState *state, Trace *trace, u64 code_header_ptr, u64 pc_offset, bool return_address) { - return _push_with_return_address( - trace, code_header_ptr, pc_offset, FRAME_MARKER_DOTNET, return_address); + const u8 ra_flag = return_address ? FRAME_FLAG_RETURN_ADDRESS : 0; + + u64 *data = + push_frame(state, trace, FRAME_MARKER_DOTNET, FRAME_FLAG_PID_SPECIFIC | ra_flag, pc_offset, 1); + if (!data) { + return ERR_STACK_LENGTH_EXCEEDED; + } + data[0] = code_header_ptr; + return ERR_OK; } // Unwind one dotnet frame @@ -247,7 +254,7 @@ static EBPF_INLINE ErrorCode unwind_one_dotnet_frame(PerCPURecord *record) if (error != ERR_DOTNET_CODE_TOO_LARGE) { return error; } - return _push(trace, 0, ERR_DOTNET_CODE_TOO_LARGE, FRAME_MARKER_DOTNET | FRAME_MARKER_ERROR_BIT); + return push_error(state, trace, FRAME_MARKER_DOTNET, ERR_DOTNET_CODE_TOO_LARGE); } // code_start points to beginning of the JIT generated code. This is preceded by a CodeHeader @@ -268,7 +275,7 @@ static EBPF_INLINE ErrorCode unwind_one_dotnet_frame(PerCPURecord *record) (unsigned long)code_start, (unsigned long)code_header_ptr, (unsigned long)(pc - code_start)); - error = push_dotnet(trace, (code_header_ptr << 5) + type, pc - code_start, return_address); + error = push_dotnet(state, trace, (code_header_ptr << 5) + type, pc - code_start, return_address); if (error) { return error; } @@ -289,7 +296,7 @@ static EBPF_INLINE int unwind_dotnet(struct pt_regs *ctx) Trace *trace = &record->trace; u32 pid = trace->pid; - DEBUG_PRINT("==== unwind_dotnet %d ====", trace->stack_len); + DEBUG_PRINT("==== unwind_dotnet %d ====", trace->num_frames); int unwinder = PROG_UNWIND_STOP; ErrorCode error = ERR_OK; diff --git a/support/ebpf/extmaps.h b/support/ebpf/extmaps.h index 30ae2a7af..1aa62bb05 100644 --- a/support/ebpf/extmaps.h +++ b/support/ebpf/extmaps.h @@ -14,7 +14,6 @@ extern struct reported_pids_t reported_pids; extern struct pid_events_t pid_events; extern struct inhibit_events_t inhibit_events; extern struct interpreter_offsets_t interpreter_offsets; -extern struct system_config_t system_config; extern struct trace_events_t trace_events; extern struct go_labels_procs_t go_labels_procs; diff --git a/support/ebpf/frametypes.h b/support/ebpf/frametypes.h index fd9f913d2..c418dcac0 100644 --- a/support/ebpf/frametypes.h +++ b/support/ebpf/frametypes.h @@ -7,10 +7,6 @@ #ifndef OPTI_FRAMETYPES_H #define OPTI_FRAMETYPES_H -// Defines the bit mask that, when ORed with it, turn any of the below -// frame types into an error frame. -#define FRAME_MARKER_ERROR_BIT 0x80 - // Indicates that the interpreter/runtime this frame belongs to is unknown. #define FRAME_MARKER_UNKNOWN 0x0 // Indicates a Python frame @@ -38,9 +34,14 @@ // Indicates a BEAM frame #define FRAME_MARKER_BEAM 0xC -// Indicates a frame containing information about a critical unwinding error -// that caused further unwinding to be aborted. -#define FRAME_MARKER_ABORT (0x7F | FRAME_MARKER_ERROR_BIT) +// Frame flags +// Indicates that this frame is an error frame. +#define FRAME_FLAG_ERROR (1U << 0) +// Indicates that this frame PC is a return address. +#define FRAME_FLAG_RETURN_ADDRESS (1U << 1) +// Indicates that the frame contains process specific addresses, +// and the PID should be included in the caching key. +#define FRAME_FLAG_PID_SPECIFIC (1U << 2) // HotSpot frame subtypes stored in a bitfield of the trace->lines[] #define FRAME_HOTSPOT_STUB 0 diff --git a/support/ebpf/hotspot_tracer.ebpf.c b/support/ebpf/hotspot_tracer.ebpf.c index 93263dc1b..94794111b 100644 --- a/support/ebpf/hotspot_tracer.ebpf.c +++ b/support/ebpf/hotspot_tracer.ebpf.c @@ -101,9 +101,19 @@ struct hotspot_procs_t { } hotspot_procs SEC(".maps"); // Record a HotSpot frame -static EBPF_INLINE ErrorCode push_hotspot(Trace *trace, u64 file, u64 line, bool return_address) +static EBPF_INLINE ErrorCode +push_hotspot(UnwindState *state, Trace *trace, u64 file, u64 line, bool return_address) { - return _push_with_return_address(trace, file, line, FRAME_MARKER_HOTSPOT, return_address); + const u8 ra_flag = return_address ? FRAME_FLAG_RETURN_ADDRESS : 0; + + u64 *data = + push_frame(state, trace, FRAME_MARKER_HOTSPOT, FRAME_FLAG_PID_SPECIFIC | ra_flag, 0, 2); + if (!data) { + return ERR_STACK_LENGTH_EXCEEDED; + } + data[0] = file; + data[1] = line; + return ERR_OK; } // calc_line merges the three values to be encoded in a frame 'line' @@ -207,11 +217,7 @@ hotspot_handle_vtable_chunks(HotspotUnwindInfo *ui, HotspotUnwindAction *action) } static EBPF_INLINE ErrorCode hotspot_handle_interpreter( - UnwindState *state, - Trace *trace, - HotspotUnwindInfo *ui, - HotspotProcInfo *ji, - HotspotUnwindAction *action) + UnwindState *state, HotspotUnwindInfo *ui, HotspotProcInfo *ji, HotspotUnwindAction *action) { // Hotspot Interpreter has it's custom stack layout, and the unwinding is done based // on frame pointer. No frame information is in the CodeBlob header. @@ -246,7 +252,7 @@ static EBPF_INLINE ErrorCode hotspot_handle_interpreter( } u64 bcp; - if (trace->stack_len) { + if (state->return_address) { // Interpreter frame has the BCP value stored if (ji->new_bcp_slot) { // JDK9+ frame has new 'mirror' slot which offsets the BCP slot @@ -770,7 +776,7 @@ static EBPF_INLINE ErrorCode hotspot_execute_unwind_action( case UA_UNWIND_COMPLETE: { unwind_complete:; u64 line = calc_line(ui->line.subtype, ui->line.pc_delta_or_bci, ui->line.ptr_check); - ErrorCode error = push_hotspot(trace, ui->file, line, state->return_address); + ErrorCode error = push_hotspot(state, trace, ui->file, line, state->return_address); if (error) { return error; } @@ -896,7 +902,7 @@ hotspot_unwind_one_frame(PerCPURecord *record, HotspotProcInfo *ji, bool maybe_t &cbi, trace, &ui, ji, &action, maybe_topmost && !state->return_address); break; case FRAMETYPE_Interpreter: // main Interpreter program running byte code - err = hotspot_handle_interpreter(state, trace, &ui, ji, &action); + err = hotspot_handle_interpreter(state, &ui, ji, &action); break; case FRAMETYPE_vtable_chunks: // megamorphic interface call site err = hotspot_handle_vtable_chunks(&ui, &action); @@ -923,7 +929,7 @@ static EBPF_INLINE int unwind_hotspot(struct pt_regs *ctx) Trace *trace = &record->trace; pid_t pid = trace->pid; - DEBUG_PRINT("==== jvm: unwind %d ====", trace->stack_len); + DEBUG_PRINT("==== jvm: unwind %d ====", trace->num_frames); int unwinder = PROG_UNWIND_STOP; ErrorCode error = ERR_OK; diff --git a/support/ebpf/integration_test.ebpf.c b/support/ebpf/integration_test.ebpf.c index 4bd17ac36..ced8b3139 100644 --- a/support/ebpf/integration_test.ebpf.c +++ b/support/ebpf/integration_test.ebpf.c @@ -28,56 +28,17 @@ static EBPF_INLINE void send_sample_traces(void *ctx, u64 pid, s32 kstack) trace->pid = pid; trace->tid = pid; trace->kernel_stack_id = -1; - trace->stack_len = 1; - trace->frames[0] = (Frame){ - .kind = FRAME_MARKER_NATIVE, - .file_id = 1337, - .addr_or_line = 21, - }; + + u64 *data = push_frame(&record->state, trace, FRAME_MARKER_NATIVE, 0, 21, 1); + if (data) { + data[0] = 1337; + } send_trace(ctx, trace); // Single native frame, with kernel trace. trace->comm[3] = 2; trace->kernel_stack_id = kstack; send_trace(ctx, trace); - - // Single Python frame. - trace->comm[3] = 3; - trace->kernel_stack_id = -1; - trace->stack_len = 3; - trace->frames[0] = (Frame){ - .kind = FRAME_MARKER_NATIVE, - .file_id = 1337, - .addr_or_line = 42, - }; - trace->frames[1] = (Frame){ - .kind = FRAME_MARKER_NATIVE, - .file_id = 1338, - .addr_or_line = 21, - }; - trace->frames[2] = (Frame){ - .kind = FRAME_MARKER_PYTHON, - .file_id = 1339, - .addr_or_line = 22, - }; - send_trace(ctx, trace); - - // Maximum length native trace. - trace->comm[3] = 4; - trace->stack_len = MAX_FRAME_UNWINDS; - trace->kernel_stack_id = kstack; - UNROLL for (u64 i = 0; i < MAX_FRAME_UNWINDS; ++i) - { - // NOTE: this init schema eats up a lot of instructions. If we need more - // space later, we can instead just init `.kind` and a few fields in the - // start, middle, and end of the trace. - trace->frames[i] = (Frame){ - .kind = FRAME_MARKER_NATIVE, - .file_id = ~i, - .addr_or_line = i, - }; - } - send_trace(ctx, trace); } // tracepoint_integration__sched_switch fetches the current kernel stack ID from diff --git a/support/ebpf/interpreter_dispatcher.ebpf.c b/support/ebpf/interpreter_dispatcher.ebpf.c index a8b9ee85d..6d6509e36 100644 --- a/support/ebpf/interpreter_dispatcher.ebpf.c +++ b/support/ebpf/interpreter_dispatcher.ebpf.c @@ -257,7 +257,7 @@ static EBPF_INLINE int unwind_stop(struct pt_regs *ctx) // If the stack is otherwise empty, push an error for that: we should // never encounter empty stacks for successful unwinding. - if (trace->stack_len == 0 && trace->kernel_stack_id < 0) { + if (trace->frame_data_len == 0 && trace->kernel_stack_id < 0) { DEBUG_PRINT("unwind_stop called but the stack is empty"); increment_metric(metricID_ErrEmptyStack); if (!state->unwind_error) { @@ -268,7 +268,7 @@ static EBPF_INLINE int unwind_stop(struct pt_regs *ctx) // If unwinding was aborted due to a critical error, push an error frame. if (state->unwind_error) { DEBUG_PRINT("Aborting further unwinding due to error code %d", state->unwind_error); - push_error(&record->trace, state->unwind_error); + push_abort(trace, state->unwind_error); } switch (state->error_metric) { @@ -280,7 +280,7 @@ static EBPF_INLINE int unwind_stop(struct pt_regs *ctx) if (report_pid(ctx, pid_tgid, record->ratelimitAction)) { increment_metric(metricID_NumUnknownPC); } - // Fallthrough to report the error + // fallthrough default: increment_metric(state->error_metric); } @@ -294,7 +294,7 @@ static EBPF_INLINE int unwind_stop(struct pt_regs *ctx) // through different data structures, we'd have to keep a list of known empty traces to // also prevent the corresponding trace counts to be sent out. OTOH, if we do it here, // this is trivial. - if (trace->stack_len == 1 && trace->kernel_stack_id < 0 && state->unwind_error) { + if (trace->frame_data_len == 1 && trace->kernel_stack_id < 0 && state->unwind_error) { if (filter_error_frames) { return 0; } diff --git a/support/ebpf/native_stack_trace.ebpf.c b/support/ebpf/native_stack_trace.ebpf.c index f22ce4fc8..85727ff2f 100644 --- a/support/ebpf/native_stack_trace.ebpf.c +++ b/support/ebpf/native_stack_trace.ebpf.c @@ -108,9 +108,17 @@ struct kernel_stackmap_t { } kernel_stackmap SEC(".maps"); // Record a native frame -static EBPF_INLINE ErrorCode push_native(Trace *trace, u64 file, u64 line, bool return_address) +static EBPF_INLINE ErrorCode +push_native(UnwindState *state, Trace *trace, u64 file, u64 line, bool return_address) { - return _push_with_return_address(trace, file, line, FRAME_MARKER_NATIVE, return_address); + const u8 ra_flag = return_address ? FRAME_FLAG_RETURN_ADDRESS : 0; + + u64 *data = push_frame(state, trace, FRAME_MARKER_NATIVE, ra_flag, line, 1); + if (!data) { + return ERR_STACK_LENGTH_EXCEEDED; + } + data[0] = file; + return ERR_OK; } // A single step for the bsearch into the big_stack_deltas array. This is really a textbook bsearch @@ -619,8 +627,7 @@ static EBPF_INLINE int unwind_native(struct pt_regs *ctx) unwinder = PROG_UNWIND_STOP; // Unwind native code - u32 frame_idx = trace->stack_len; - DEBUG_PRINT("==== unwind_native %d ====", frame_idx); + DEBUG_PRINT("==== unwind_native %d ====", trace->num_frames); increment_metric(metricID_UnwindNativeAttempts); // Push frame first. The PC is valid because a text section mapping was found. @@ -628,8 +635,9 @@ static EBPF_INLINE int unwind_native(struct pt_regs *ctx) "Pushing %llx %llx to position %u on stack", record->state.text_section_id, record->state.text_section_offset, - trace->stack_len); + trace->num_frames); error = push_native( + &record->state, trace, record->state.text_section_id, record->state.text_section_offset, diff --git a/support/ebpf/perl_tracer.ebpf.c b/support/ebpf/perl_tracer.ebpf.c index 5c2846051..16af5d83f 100644 --- a/support/ebpf/perl_tracer.ebpf.c +++ b/support/ebpf/perl_tracer.ebpf.c @@ -75,10 +75,17 @@ struct perl_procs_t { } perl_procs SEC(".maps"); // Record a Perl frame -static EBPF_INLINE ErrorCode push_perl(Trace *trace, u64 file, u64 line) +static EBPF_INLINE ErrorCode push_perl(UnwindState *state, Trace *trace, u64 file, u64 line) { DEBUG_PRINT("Pushing perl frame cop=0x%lx, cv=0x%lx", (unsigned long)file, (unsigned long)line); - return _push(trace, file, line, FRAME_MARKER_PERL); + + u64 *data = push_frame(state, trace, FRAME_MARKER_PERL, FRAME_FLAG_PID_SPECIFIC, 0, 2); + if (!data) { + return ERR_STACK_LENGTH_EXCEEDED; + } + data[0] = file; + data[1] = line; + return ERR_OK; } // resolve_cv_egv() takes in a CV* and follows the pointers to resolve this CV's @@ -223,7 +230,7 @@ process_perl_frame(PerCPURecord *record, const PerlProcInfo *perlinfo, const voi if (!egv) { goto err; } - if (push_perl(trace, (u64)egv, (u64)record->perlUnwindState.cop) != ERR_OK) { + if (push_perl(&record->state, trace, (u64)egv, (u64)record->perlUnwindState.cop) != ERR_OK) { return PROG_UNWIND_STOP; } record->perlUnwindState.cop = 0; @@ -323,7 +330,7 @@ static EBPF_INLINE int walk_perl_stack(PerCPURecord *record, const PerlProcInfo u64 cop = (u64)record->perlUnwindState.cop; if (cop) { DEBUG_PRINT("End of perl stack - pushing main 0x%lx", (unsigned long)cop); - if (push_perl(trace, 0, cop) != ERR_OK) { + if (push_perl(&record->state, trace, 0, cop) != ERR_OK) { return PROG_UNWIND_STOP; } record->perlUnwindState.cop = 0; diff --git a/support/ebpf/php_tracer.ebpf.c b/support/ebpf/php_tracer.ebpf.c index 83c6998a2..b526ced01 100644 --- a/support/ebpf/php_tracer.ebpf.c +++ b/support/ebpf/php_tracer.ebpf.c @@ -26,16 +26,24 @@ struct php_procs_t { } php_procs SEC(".maps"); // Record a PHP frame -static EBPF_INLINE ErrorCode push_php(Trace *trace, u64 file, u64 line, bool is_jitted) +static EBPF_INLINE ErrorCode +push_php(UnwindState *state, Trace *trace, u64 file, u64 line, bool is_jitted) { - int frame_type = is_jitted ? FRAME_MARKER_PHP_JIT : FRAME_MARKER_PHP; - return _push(trace, file, line, frame_type); + u8 frame_type = is_jitted ? FRAME_MARKER_PHP_JIT : FRAME_MARKER_PHP; + + u64 *data = push_frame(state, trace, frame_type, FRAME_FLAG_PID_SPECIFIC, 0, 2); + if (!data) { + return ERR_STACK_LENGTH_EXCEEDED; + } + data[0] = file; + data[1] = line; + return ERR_OK; } // Record a PHP call for which no function object is available -static EBPF_INLINE ErrorCode push_unknown_php(Trace *trace) +static EBPF_INLINE ErrorCode push_unknown_php(UnwindState *state, Trace *trace) { - return _push(trace, UNKNOWN_FILE, FUNC_TYPE_UNKNOWN, FRAME_MARKER_PHP); + return push_php(state, trace, UNKNOWN_FILE, FUNC_TYPE_UNKNOWN, false); } static EBPF_INLINE int process_php_frame( @@ -59,7 +67,7 @@ static EBPF_INLINE int process_php_frame( // It is possible there is no function object. if (!zend_function) { - if (push_unknown_php(trace) != ERR_OK) { + if (push_unknown_php(&record->state, trace) != ERR_OK) { DEBUG_PRINT("failed to push unknown php frame"); return -1; } @@ -109,7 +117,9 @@ static EBPF_INLINE int process_php_frame( u64 lineno_and_type_info = ((u64)*type_info) << 32 | lineno; DEBUG_PRINT("Pushing PHP 0x%lx %u", (unsigned long)zend_function, lineno); - if (push_php(trace, (u64)zend_function, lineno_and_type_info, is_jitted) != ERR_OK) { + if ( + push_php(&record->state, trace, (u64)zend_function, lineno_and_type_info, is_jitted) != + ERR_OK) { DEBUG_PRINT("failed to push php frame"); return -1; } diff --git a/support/ebpf/python_tracer.ebpf.c b/support/ebpf/python_tracer.ebpf.c index e6f52e41e..de2c11734 100644 --- a/support/ebpf/python_tracer.ebpf.c +++ b/support/ebpf/python_tracer.ebpf.c @@ -25,9 +25,15 @@ struct py_procs_t { } py_procs SEC(".maps"); // Record a Python frame -static EBPF_INLINE ErrorCode push_python(Trace *trace, u64 file, u64 line) +static EBPF_INLINE ErrorCode push_python(UnwindState *state, Trace *trace, u64 file, u64 line) { - return _push(trace, file, line, FRAME_MARKER_PYTHON); + u64 *data = push_frame(state, trace, FRAME_MARKER_PYTHON, FRAME_FLAG_PID_SPECIFIC, 0, 2); + if (!data) { + return ERR_STACK_LENGTH_EXCEEDED; + } + data[0] = file; + data[1] = line; + return ERR_OK; } static EBPF_INLINE u64 py_encode_lineno(u32 object_id, u32 f_lasti) @@ -154,7 +160,7 @@ static EBPF_INLINE ErrorCode process_python_frame( push_frame: DEBUG_PRINT("Pushing Python %lx %lu", (unsigned long)file_id, (unsigned long)lineno); - ErrorCode error = push_python(trace, file_id, lineno); + ErrorCode error = push_python(&record->state, trace, file_id, lineno); if (error) { DEBUG_PRINT("failed to push python frame"); return error; diff --git a/support/ebpf/ruby_tracer.ebpf.c b/support/ebpf/ruby_tracer.ebpf.c index 71b5b1b20..c88bc5817 100644 --- a/support/ebpf/ruby_tracer.ebpf.c +++ b/support/ebpf/ruby_tracer.ebpf.c @@ -26,9 +26,15 @@ struct ruby_procs_t { #define RUBY_FRAME_FLAG_LAMBDA 0x0100 // Record a Ruby frame -static EBPF_INLINE ErrorCode push_ruby(Trace *trace, u64 file, u64 line) +static EBPF_INLINE ErrorCode push_ruby(UnwindState *state, Trace *trace, u64 file, u64 line) { - return _push(trace, file, line, FRAME_MARKER_RUBY); + u64 *data = push_frame(state, trace, FRAME_MARKER_RUBY, FRAME_FLAG_PID_SPECIFIC, 0, 2); + if (!data) { + return ERR_STACK_LENGTH_EXCEEDED; + } + data[0] = file; + data[1] = line; + return ERR_OK; } // walk_ruby_stack processes a Ruby VM stack, extracts information from the individual frames and @@ -200,7 +206,7 @@ static EBPF_INLINE ErrorCode walk_ruby_stack( // For symbolization of the frame we forward the information about the instruction sequence // and program counter to user space. // From this we can then extract information like file or function name and line number. - ErrorCode error = push_ruby(trace, (u64)iseq_body, pc); + ErrorCode error = push_ruby(&record->state, trace, (u64)iseq_body, pc); if (error) { DEBUG_PRINT("ruby: failed to push frame"); return error; diff --git a/support/ebpf/tracemgmt.h b/support/ebpf/tracemgmt.h index 8c6d8c8a6..2e812089f 100644 --- a/support/ebpf/tracemgmt.h +++ b/support/ebpf/tracemgmt.h @@ -251,7 +251,8 @@ static inline EBPF_INLINE PerCPURecord *get_pristine_per_cpu_record() Trace *trace = &record->trace; trace->kernel_stack_id = -1; - trace->stack_len = 0; + trace->frame_data_len = 0; + trace->num_frames = 0; trace->pid = 0; trace->tid = 0; @@ -320,72 +321,76 @@ static inline EBPF_INLINE bool unwinder_unwind_frame_pointer(UnwindState *state) return true; } -// Push the file ID, line number and frame type into FrameList with a user-defined -// maximum stack size. -// -// NOTE: The line argument is used for a lot of different purposes, depending on -// the frame type. For example error frames use it to store the error number, -// and hotspot puts a subtype and BCI indices, amongst other things (see -// calc_line). This should probably be renamed to something like "frame type -// specific data". -static inline EBPF_INLINE ErrorCode _push_with_max_frames( - Trace *trace, u64 file, u64 line, u8 frame_type, u8 return_address, u32 max_frames) +static inline EBPF_INLINE u64 frame_header(u8 frame_type, u8 flags, u8 length, u64 data) { - if (trace->stack_len >= max_frames) { - DEBUG_PRINT("unable to push frame: stack is full"); - increment_metric(metricID_UnwindErrStackLengthExceeded); - return ERR_STACK_LENGTH_EXCEEDED; - } - -#ifdef TESTING_COREDUMP - // tools/coredump uses CGO to build the eBPF code. This dispatches - // the frame information directly to helper implemented in ebpfhelpers.go. - int __push_frame(u64, u64, u64, u8, u8); - trace->stack_len++; - return __push_frame(__cgo_ctx->id, file, line, frame_type, return_address); -#else - trace->frames[trace->stack_len++] = (Frame){ - .file_id = file, - .addr_or_line = line, - .kind = frame_type, - .return_address = return_address, - }; - - return ERR_OK; -#endif + // frame header format (fixed size): + // #bits usage + // 4 frame type + // 4 frame flags + // 4 number of 64-bit 'variable' fields + // 52 type specific data + return ((u64)frame_type << 60) | ((u64)flags << 56) | ((u64)length << 52) | data; } -// Push the file ID, line number and frame type into FrameList -static inline EBPF_INLINE ErrorCode -_push_with_return_address(Trace *trace, u64 file, u64 line, u8 frame_type, bool return_address) +// Push a data frame with variable length payload. This function allocates space from +// the 'trace' for one frame and populates a common header for it. Frame type and flags +// are used to determine the symbolization plugin and how to cache and interpret it. +// The header has a 52 bit 'data' field for use of the interpreter, along with variable +// number of 64-bit 'variable' fields. +// On success, a pointer to the first 'variable' field is returned. +// On failure, NULL is returned. The 'UnwindState' is updated for too long stack error. +static inline EBPF_INLINE u64 *push_frame( + UnwindState *state, Trace *trace, u8 frame_type, u8 frame_flags, u64 frame_data, u8 frame_varlen) { - return _push_with_max_frames( - trace, file, line, frame_type, return_address, MAX_NON_ERROR_FRAME_UNWINDS); + const int max_frame_size = sizeof trace->frame_data / sizeof trace->frame_data[0]; + const int error_frame_size = 1; + + // Check that there is enough space for this frame and at least one error frame. + u64 *pos = &trace->frame_data[trace->frame_data_len]; + u8 frame_size = frame_varlen + 1; + if (pos >= &trace->frame_data[max_frame_size - error_frame_size - frame_size]) { + state->error_metric = metricID_UnwindErrStackLengthExceeded; + return NULL; + } + trace->num_frames++; + trace->frame_data_len += frame_size; + pos[0] = frame_header(frame_type, frame_flags, frame_size, frame_data); + return &pos[1]; } -// Push the file ID, line number and frame type into FrameList -static inline EBPF_INLINE ErrorCode _push(Trace *trace, u64 file, u64 line, u8 frame_type) +// Push an interpreter specific error frame. +static inline EBPF_INLINE ErrorCode +push_error(UnwindState *state, Trace *trace, u8 frame_type, ErrorCode error) { - return _push_with_max_frames(trace, file, line, frame_type, 0, MAX_NON_ERROR_FRAME_UNWINDS); + u64 *data = push_frame(state, trace, frame_type, FRAME_FLAG_ERROR, error, 0); + if (data) { + return ERR_OK; + } + return ERR_STACK_LENGTH_EXCEEDED; } // Push a critical error frame. -static inline EBPF_INLINE ErrorCode push_error(Trace *trace, ErrorCode error) +static inline EBPF_INLINE void push_abort(Trace *trace, ErrorCode error) { - return _push_with_max_frames(trace, 0, error, FRAME_MARKER_ABORT, 0, MAX_FRAME_UNWINDS); + const int max_frame_size = sizeof trace->frame_data / sizeof trace->frame_data[0]; + + // Check that there is enough space for this frame and at least one error frame. + if (trace->frame_data_len < max_frame_size) { + trace->num_frames++; + trace->frame_data[trace->frame_data_len++] = + frame_header(FRAME_MARKER_UNKNOWN, FRAME_FLAG_ERROR, 1, error); + } } // Send a trace to user-land via the `trace_events` perf event buffer. static inline EBPF_INLINE void send_trace(void *ctx, Trace *trace) { - const u64 num_empty_frames = (MAX_FRAME_UNWINDS - trace->stack_len); - const u64 send_size = sizeof(Trace) - sizeof(Frame) * num_empty_frames; + const u64 send_size = sizeof(Trace) - sizeof(trace->frame_data) + + sizeof(trace->frame_data[0]) * trace->frame_data_len; - if (send_size > sizeof(Trace)) { - return; // unreachable + if (send_size < sizeof(Trace)) { + bpf_perf_event_output(ctx, &trace_events, BPF_F_CURRENT_CPU, trace, send_size); } - - bpf_perf_event_output(ctx, &trace_events, BPF_F_CURRENT_CPU, trace, send_size); } // is_kernel_address checks if the given address looks like virtual address to kernel memory. @@ -506,7 +511,7 @@ get_next_unwinder_after_native_frame(PerCPURecord *record, int *unwinder) return ERR_NATIVE_ZERO_PC; } - DEBUG_PRINT("==== Resolve next frame unwinder: frame %d ====", record->trace.stack_len); + DEBUG_PRINT("==== Resolve next frame unwinder: frame %d ====", record->trace.num_frames); ErrorCode error = resolve_unwind_mapping(record, unwinder); if (error) { return error; diff --git a/support/ebpf/tracer.ebpf.amd64 b/support/ebpf/tracer.ebpf.amd64 index 789cf0c9e..0d8390a77 100644 Binary files a/support/ebpf/tracer.ebpf.amd64 and b/support/ebpf/tracer.ebpf.amd64 differ diff --git a/support/ebpf/tracer.ebpf.arm64 b/support/ebpf/tracer.ebpf.arm64 index f0dd60b23..c9c9e34f0 100644 Binary files a/support/ebpf/tracer.ebpf.arm64 and b/support/ebpf/tracer.ebpf.arm64 differ diff --git a/support/ebpf/types.h b/support/ebpf/types.h index 08c1774bc..b8eac5583 100644 --- a/support/ebpf/types.h +++ b/support/ebpf/types.h @@ -348,48 +348,11 @@ typedef enum TraceOrigin { TRACE_PROBE, } TraceOrigin; -// MAX_FRAME_UNWINDS defines the maximum number of frames per -// Trace we can unwind and respect the limit of eBPF instructions, -// limit of tail calls and limit of stack size per eBPF program. -#define MAX_FRAME_UNWINDS 128 - -// MAX_NON_ERROR_FRAME_UNWINDS defines the maximum number of frames -// to be pushed by unwinders while still leaving space for an error frame. -// This is used to make sure that there is always space for an error -// frame reporting that we ran out of stack space. -#define MAX_NON_ERROR_FRAME_UNWINDS (MAX_FRAME_UNWINDS - 1) - // Maximum number of unique stack deltas needed on a system. This is based on // normal desktop /usr/bin/* and /usr/lib/*.so having about 9700 unique deltas. // Can be increased up to 2^15, see also STACK_DELTA_COMMAND_FLAG. #define UNWIND_INFO_MAX_ENTRIES 16384 -// Type to represent a globally-unique file id to be used as key for a BPF hash map -typedef u64 FileID; - -// Individual frame in a stack-trace. -typedef struct Frame { - // IDs that uniquely identify a file combination - FileID file_id; - // For PHP this is the line numbers, corresponding to the files in `stack`. - // For Python, each value provides information to allow for the recovery of - // the line number associated with its corresponding offset in `stack`. - // The lower 32 bits provide the co_firstlineno value and the upper 32 bits - // provide the f_lasti value. Other interpreter handlers use the field in - // a similarly domain-specific fashion. - u64 addr_or_line; - // Indicates the type of the frame (Python, PHP, native etc.). - u8 kind; - // Indicates that the address is a return address. - u8 return_address; - // Explicit padding bytes that the compiler would have inserted anyway. - // Here to make it clear to readers that there are spare bytes that could - // be put to work without extra cost in case an interpreter needs it. - u8 pad[6]; -} Frame; - -_Static_assert(sizeof(Frame) == 3 * 8, "frame padding not working as expected"); - // TSDInfo contains data needed to extract Thread Specific Data (TSD) values typedef struct TSDInfo { // Offset is the pointer difference from "tpbase" pointer to the C-library @@ -597,8 +560,10 @@ typedef struct Trace { CustomLabelsArray custom_labels; // The kernel stack ID. s32 kernel_stack_id; - // The number of frames in the stack. - u32 stack_len; + // The number of frame_data elements present. + u16 frame_data_len; + // The number of frames present. + u16 num_frames; // origin indicates the source of the trace. TraceOrigin origin; @@ -606,14 +571,22 @@ typedef struct Trace { // offtime stores the nanoseconds that the trace was off-cpu for. u64 offtime; - // The frames of the stack trace. - Frame frames[MAX_FRAME_UNWINDS]; + // The frame data of the stack trace. Each frame is variable length. + // Frame is currently 2-3 entries long. This array size limits the + // number of frames we can unwind, but also increases the memory + // needed for buffering everything. The 3kB entries here is chosen + // to allow about 1024 frames in a trace to be sent. + u64 frame_data[3072]; - // NOTE: both send_trace in BPF and loadBpfTrace in UM code require `frames` - // to be the last item in the struct. Do not add new members here without also - // adjusting the UM code. + // NOTE: both send_trace in BPF and loadBpfTrace in UM code require `frame_data` + // to be the last item in the struct. When sending as a perf event, only the + // 'frame_data_len' elements of 'frame_data' are sent. } Trace; +// Trace is sent as a perf raw event. As all perf events are contained within +// struct perf_event_header with 'u16 size', this limits the size of Trace. +_Static_assert(sizeof(struct Trace) < 63 * 1024, "Trace too large"); + // Container for unwinding state typedef struct UnwindState { // Current register value for Program Counter @@ -805,6 +778,9 @@ typedef struct PerCPURecord { u8 ratelimitAction; } PerCPURecord; +// https://github.com/torvalds/linux/blob/e9a6fb0bcdd7609be6969112f3fbfcce3b1d4a7c/include/linux/percpu.h#L24C39-L24C47 +_Static_assert(sizeof(struct PerCPURecord) <= (32 << 10), "Per CPU record too large"); + // UnwindInfo contains the unwind information needed to unwind one frame // from a specific address. typedef struct UnwindInfo { diff --git a/support/ebpf/v8_tracer.ebpf.c b/support/ebpf/v8_tracer.ebpf.c index 867f2a67b..c346d36a4 100644 --- a/support/ebpf/v8_tracer.ebpf.c +++ b/support/ebpf/v8_tracer.ebpf.c @@ -39,14 +39,22 @@ struct v8_procs_t { // Record a V8 frame static EBPF_INLINE ErrorCode push_v8( - Trace *trace, unsigned long pointer_and_type, unsigned long delta_or_marker, bool return_address) + UnwindState *state, Trace *trace, u64 pointer_and_type, u64 delta_or_marker, bool return_address) { DEBUG_PRINT( - "Pushing v8 frame delta_or_marker=%lx, pointer_and_type=%lx", + "Pushing v8 frame delta_or_marker=%llx, pointer_and_type=%llx", delta_or_marker, pointer_and_type); - return _push_with_return_address( - trace, pointer_and_type, delta_or_marker, FRAME_MARKER_V8, return_address); + + const u8 ra_flag = return_address ? FRAME_FLAG_RETURN_ADDRESS : 0; + + u64 *data = push_frame(state, trace, FRAME_MARKER_V8, FRAME_FLAG_PID_SPECIFIC | ra_flag, 0, 2); + if (!data) { + return ERR_STACK_LENGTH_EXCEEDED; + } + data[0] = pointer_and_type; + data[1] = delta_or_marker; + return ERR_OK; } // Verify a V8 tagged pointer @@ -224,7 +232,7 @@ static EBPF_INLINE ErrorCode unwind_one_v8_frame(PerCPURecord *record, V8ProcInf // - the JSFunction's Code object was changed due to On-Stack-Replacement or // or other deoptimization reasons. This case is currently not handled. - if (top && trace->stack_len == 0) { + if (top && !state->return_address) { unsigned long stk[3]; if (bpf_probe_read_user(stk, sizeof(stk), (void *)(sp - sizeof(stk)))) { DEBUG_PRINT("v8: --> bad stack pointer"); @@ -268,7 +276,7 @@ static EBPF_INLINE ErrorCode unwind_one_v8_frame(PerCPURecord *record, V8ProcInf delta_or_marker = (pc - code_start) | ((uintptr_t)cookie << V8_LINE_COOKIE_SHIFT); frame_done:; - ErrorCode error = push_v8(trace, pointer_and_type, delta_or_marker, state->return_address); + ErrorCode error = push_v8(state, trace, pointer_and_type, delta_or_marker, state->return_address); if (error) { return error; } @@ -309,7 +317,7 @@ static EBPF_INLINE int unwind_v8(struct pt_regs *ctx) Trace *trace = &record->trace; u32 pid = trace->pid; - DEBUG_PRINT("==== unwind_v8 %d ====", trace->stack_len); + DEBUG_PRINT("==== unwind_v8 %d ====", trace->num_frames); int unwinder = PROG_UNWIND_STOP; ErrorCode error = ERR_OK; diff --git a/support/types.go b/support/types.go index 0bba25e7c..ea9832094 100644 --- a/support/types.go +++ b/support/types.go @@ -11,21 +11,25 @@ import ( ) const ( - FrameMarkerUnknown = 0x0 - FrameMarkerErrorBit = 0x80 - FrameMarkerPython = 0x1 - FrameMarkerNative = 0x3 - FrameMarkerPHP = 0x2 - FrameMarkerPHPJIT = 0x9 - FrameMarkerKernel = 0x4 - FrameMarkerHotSpot = 0x5 - FrameMarkerRuby = 0x6 - FrameMarkerPerl = 0x7 - FrameMarkerV8 = 0x8 - FrameMarkerDotnet = 0xa - FrameMarkerGo = 0xb - FrameMarkerBEAM = 0xc - FrameMarkerAbort = 0xff + FrameMarkerUnknown = 0x0 + FrameMarkerPython = 0x1 + FrameMarkerNative = 0x3 + FrameMarkerPHP = 0x2 + FrameMarkerPHPJIT = 0x9 + FrameMarkerKernel = 0x4 + FrameMarkerHotSpot = 0x5 + FrameMarkerRuby = 0x6 + FrameMarkerPerl = 0x7 + FrameMarkerV8 = 0x8 + FrameMarkerDotnet = 0xa + FrameMarkerBEAM = 0xc + FrameMarkerGo = 0xb +) + +const ( + FrameFlagError = 0x1 + FrameFlagReturnAddress = 0x2 + FrameFlagPidSpecific = 0x4 ) const ( @@ -52,8 +56,6 @@ const ( EventTypeGenericPID = 0x1 ) -const MaxFrameUnwinds = 0x80 - const UnwindInfoMaxEntries = 0x4000 const ( @@ -107,13 +109,6 @@ type CustomLabelsArray struct { type Event struct { Type uint32 } -type Frame struct { - File_id uint64 - Addr_or_line uint64 - Kind uint8 - Return_address uint8 - Pad [6]uint8 -} type OffsetRange struct { Lower_offset1 uint64 Upper_offset1 uint64 @@ -164,10 +159,11 @@ type Trace struct { Apm_trace_id [16]byte Custom_labels CustomLabelsArray Kernel_stack_id int32 - Stack_len uint32 + Frame_data_len uint16 + Num_frames uint16 Origin uint32 Offtime uint64 - Frames [128]Frame + Frame_data [3072]uint64 } type UnwindInfo struct { Opcode uint8 @@ -320,9 +316,8 @@ type V8ProcInfo struct { } const ( - Sizeof_Frame = 0x18 Sizeof_StackDelta = 0x4 - Sizeof_Trace = 0xed0 + Sizeof_Trace = 0x62d0 sizeof_ApmIntProcInfo = 0x8 sizeof_DotnetProcInfo = 0x4 diff --git a/support/types_def.go b/support/types_def.go index a8bae4049..aee84d774 100644 --- a/support/types_def.go +++ b/support/types_def.go @@ -18,21 +18,25 @@ import ( import "C" const ( - FrameMarkerUnknown = C.FRAME_MARKER_UNKNOWN - FrameMarkerErrorBit = C.FRAME_MARKER_ERROR_BIT - FrameMarkerPython = C.FRAME_MARKER_PYTHON - FrameMarkerNative = C.FRAME_MARKER_NATIVE - FrameMarkerPHP = C.FRAME_MARKER_PHP - FrameMarkerPHPJIT = C.FRAME_MARKER_PHP_JIT - FrameMarkerKernel = C.FRAME_MARKER_KERNEL - FrameMarkerHotSpot = C.FRAME_MARKER_HOTSPOT - FrameMarkerRuby = C.FRAME_MARKER_RUBY - FrameMarkerPerl = C.FRAME_MARKER_PERL - FrameMarkerV8 = C.FRAME_MARKER_V8 - FrameMarkerDotnet = C.FRAME_MARKER_DOTNET - FrameMarkerGo = C.FRAME_MARKER_GO - FrameMarkerBEAM = C.FRAME_MARKER_BEAM - FrameMarkerAbort = C.FRAME_MARKER_ABORT + FrameMarkerUnknown = C.FRAME_MARKER_UNKNOWN + FrameMarkerPython = C.FRAME_MARKER_PYTHON + FrameMarkerNative = C.FRAME_MARKER_NATIVE + FrameMarkerPHP = C.FRAME_MARKER_PHP + FrameMarkerPHPJIT = C.FRAME_MARKER_PHP_JIT + FrameMarkerKernel = C.FRAME_MARKER_KERNEL + FrameMarkerHotSpot = C.FRAME_MARKER_HOTSPOT + FrameMarkerRuby = C.FRAME_MARKER_RUBY + FrameMarkerPerl = C.FRAME_MARKER_PERL + FrameMarkerV8 = C.FRAME_MARKER_V8 + FrameMarkerDotnet = C.FRAME_MARKER_DOTNET + FrameMarkerBEAM = C.FRAME_MARKER_BEAM + FrameMarkerGo = C.FRAME_MARKER_GO +) + +const ( + FrameFlagError = C.FRAME_FLAG_ERROR + FrameFlagReturnAddress = C.FRAME_FLAG_RETURN_ADDRESS + FrameFlagPidSpecific = C.FRAME_FLAG_PID_SPECIFIC ) const ( @@ -59,8 +63,6 @@ const ( EventTypeGenericPID = C.EVENT_TYPE_GENERIC_PID ) -const MaxFrameUnwinds = C.MAX_FRAME_UNWINDS - const UnwindInfoMaxEntries = C.UNWIND_INFO_MAX_ENTRIES const ( @@ -110,7 +112,6 @@ type ApmTraceID C.ApmTraceID type CustomLabel C.CustomLabel type CustomLabelsArray C.CustomLabelsArray type Event C.Event -type Frame C.Frame type OffsetRange C.OffsetRange type PIDPage C.PIDPage type PIDPageMappingInfo C.PIDPageMappingInfo @@ -134,7 +135,6 @@ type RubyProcInfo C.RubyProcInfo type V8ProcInfo C.V8ProcInfo const ( - Sizeof_Frame = C.sizeof_Frame Sizeof_StackDelta = C.sizeof_StackDelta Sizeof_Trace = C.sizeof_Trace diff --git a/tools/coredump/coredump.go b/tools/coredump/coredump.go index 63c5292c8..43dad93a4 100644 --- a/tools/coredump/coredump.go +++ b/tools/coredump/coredump.go @@ -71,7 +71,7 @@ func formatFrame(frame *libpf.Frame) (string, error) { "got invalid error code %d. forgot to `make generate`", frame.AddressOrLineno) } - if frame.Type == libpf.AbortFrame { + if frame.Type.IsAbort() { return fmt.Sprintf("", errName), nil } return fmt.Sprintf("", errName), nil diff --git a/tools/coredump/ebpfcode.h b/tools/coredump/ebpfcode.h index 56fa1b9a5..2fd9b5d81 100644 --- a/tools/coredump/ebpfcode.h +++ b/tools/coredump/ebpfcode.h @@ -66,20 +66,11 @@ int unwind_traces(u64 id, int debug, u64 tp_base, void *ctx) return cgoctx.ret; } -// We don't want to call the actual `unwind_stop` function because it'd -// require us to properly emulate all the maps required for sending frames -// to usermode. -int coredump_unwind_stop(UNUSED struct bpf_perf_event_data *ctx) +int bpf_perf_event_output( + UNUSED void *ctx, UNUSED void *map, UNUSED unsigned long long flags, void *data, UNUSED int size) { - (void)unwind_stop; - PerCPURecord *record = get_per_cpu_record(); - if (!record) - return -1; - - if (record->state.unwind_error) { - push_error(&record->trace, record->state.unwind_error); - } - + void __bpf_copy_frame(u64, void *); + __bpf_copy_frame(__cgo_ctx->id, data); return 0; } @@ -87,7 +78,7 @@ int bpf_tail_call(void *ctx, UNUSED void *map, int index) { int rc = 0; switch (index) { - case PROG_UNWIND_STOP: rc = coredump_unwind_stop(ctx); break; + case PROG_UNWIND_STOP: rc = unwind_stop(ctx); break; case PROG_UNWIND_NATIVE: rc = unwind_native(ctx); break; case PROG_UNWIND_PERL: rc = unwind_perl(ctx); break; case PROG_UNWIND_PHP: rc = unwind_php(ctx); break; diff --git a/tools/coredump/ebpfcontext.go b/tools/coredump/ebpfcontext.go index c0ddb9f0d..911618df6 100644 --- a/tools/coredump/ebpfcontext.go +++ b/tools/coredump/ebpfcontext.go @@ -6,7 +6,7 @@ package main import ( "unsafe" - "go.opentelemetry.io/ebpf-profiler/host" + "go.opentelemetry.io/ebpf-profiler/libpf" "go.opentelemetry.io/ebpf-profiler/process" "go.opentelemetry.io/ebpf-profiler/remotememory" "go.opentelemetry.io/ebpf-profiler/support" @@ -23,7 +23,7 @@ import "C" // ebpfContext is the context for EBPF code regarding the process it's unwinding. type ebpfContext struct { // trace will contain the trace from the CGO executed eBPF unwinding code - trace host.Trace + trace libpf.EbpfTrace // remotememory provides access to the target process memory space remoteMemory remotememory.RemoteMemory @@ -69,7 +69,7 @@ var ebpfContextMap = map[C.u64]*ebpfContext{} func newEBPFContext(pr process.Process) *ebpfContext { pid := pr.PID() ctx := &ebpfContext{ - trace: host.Trace{PID: pid}, + trace: libpf.EbpfTrace{PID: pid}, remoteMemory: pr.GetRemoteMemory(), PIDandTGID: C.u64(pid) << 32, pidToPageMapping: make(map[C.PIDPage]unsafe.Pointer), @@ -102,7 +102,7 @@ func (ec *ebpfContext) delMap(mapPtr unsafe.Pointer, key any) { } func (ec *ebpfContext) resetTrace() { - ec.trace.Frames = ec.trace.Frames[0:0] + ec.trace.FrameData = nil } func (ec *ebpfContext) release() { diff --git a/tools/coredump/ebpfhelpers.go b/tools/coredump/ebpfhelpers.go index 73a30d005..0ee5048b9 100644 --- a/tools/coredump/ebpfhelpers.go +++ b/tools/coredump/ebpfhelpers.go @@ -15,8 +15,7 @@ import ( "math/bits" "unsafe" - "go.opentelemetry.io/ebpf-profiler/host" - "go.opentelemetry.io/ebpf-profiler/libpf" + "go.opentelemetry.io/ebpf-profiler/libpf/pfunsafe" "go.opentelemetry.io/ebpf-profiler/support" "go.opentelemetry.io/ebpf-profiler/times" @@ -35,20 +34,6 @@ func __bpf_log(buf unsafe.Pointer, sz C.int) { log.Info(string(sliceBuffer(buf, sz))) } -//export __push_frame -func __push_frame(id, file, line C.u64, frameType, returnAddress C.uchar) C.int { - ctx := ebpfContextMap[id] - - ctx.trace.Frames = append(ctx.trace.Frames, host.Frame{ - File: host.FileID(file), - Lineno: libpf.AddressOrLineno(line), - Type: libpf.FrameType(frameType), - ReturnAddress: returnAddress != 0, - }) - - return C.ERR_OK -} - //export bpf_ktime_get_ns func bpf_ktime_get_ns() C.ulonglong { return C.ulonglong(times.GetKTime()) @@ -126,10 +111,20 @@ func __bpf_map_lookup_elem(id C.u64, mapdef unsafe.Pointer, keyptr unsafe.Pointe if deltas, ok := ctx.exeIDToStackDeltaMaps[ctx.stackDeltaFileID]; ok { return unsafe.Pointer(uintptr(deltas) + key*C.sizeof_StackDelta) } - case unsafe.Pointer(&C.metrics): + case unsafe.Pointer(&C.metrics), unsafe.Pointer(&C.report_events), + unsafe.Pointer(&C.reported_pids), unsafe.Pointer(&C.pid_events), unsafe.Pointer(&C.inhibit_events), + unsafe.Pointer(&C.apm_int_procs), unsafe.Pointer(&C.go_labels_procs): return unsafe.Pointer(uintptr(0)) default: log.Errorf("Map at 0x%x not found", mapdef) } return unsafe.Pointer(uintptr(0)) } + +//export __bpf_copy_frame +func __bpf_copy_frame(id C.u64, trace *C.Trace) { + ctx := ebpfContextMap[id] + sz := trace.frame_data_len + copy(pfunsafe.FromSlice(ctx.trace.FrameDataBuf[:sz]), pfunsafe.FromSlice(trace.frame_data[:sz])) + ctx.trace.FrameData = ctx.trace.FrameDataBuf[:sz] +} diff --git a/tracer/ebpf_integration_test.go b/tracer/ebpf_integration_test.go index 9d0f593a4..98d136d52 100644 --- a/tracer/ebpf_integration_test.go +++ b/tracer/ebpf_integration_test.go @@ -8,6 +8,7 @@ package tracer_test import ( "math" "runtime" + "slices" "sync" "testing" "time" @@ -18,7 +19,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "go.opentelemetry.io/ebpf-profiler/host" "go.opentelemetry.io/ebpf-profiler/libpf" "go.opentelemetry.io/ebpf-profiler/metrics" "go.opentelemetry.io/ebpf-profiler/rlimit" @@ -79,29 +79,10 @@ func runKernelFrameProbe(t *testing.T, tr *tracer.Tracer) { require.NoError(t, err) } -func validateTrace(t *testing.T, expected, returned *host.Trace) { - t.Helper() +type trace struct { + numKernelFrames int - require.Len(t, returned.Frames, len(expected.Frames)) - - for i, expFrame := range expected.Frames { - retFrame := returned.Frames[i] - assert.Equal(t, expFrame.File, retFrame.File) - assert.Equal(t, expFrame.Lineno, retFrame.Lineno) - assert.Equal(t, expFrame.Type, retFrame.Type) - } -} - -func generateMaxLengthTrace() host.Trace { - var trace host.Trace - for i := 0; i < support.MaxFrameUnwinds; i++ { - trace.Frames = append(trace.Frames, host.Frame{ - File: ^host.FileID(i), - Lineno: libpf.AddressOrLineno(i), - Type: support.FrameMarkerNative, - }) - } - return trace + frames libpf.EbpfFrame } func TestTraceTransmissionAndParsing(t *testing.T) { @@ -127,13 +108,13 @@ func TestTraceTransmissionAndParsing(t *testing.T) { require.NoError(t, err) defer tr.Close() - traceChan := make(chan *host.Trace, 16) + traceChan := make(chan *libpf.EbpfTrace, 16) err = tr.StartMapMonitors(ctx, traceChan) require.NoError(t, err) runKernelFrameProbe(t, tr) - traces := make(map[uint8]*host.Trace) + traces := make(map[uint8]trace) timeout := time.NewTimer(1 * time.Second) // Wait 1 second for traces to arrive. @@ -142,14 +123,20 @@ Loop: select { case <-timeout.C: break Loop - case trace := <-traceChan: - comm := trace.Comm.String() + case ebpfTrace := <-traceChan: + comm := ebpfTrace.Comm.String() require.GreaterOrEqual(t, len(comm), 4) require.Equal(t, "\xAA\xBB\xCC", comm[0:3]) - traces[comm[3]] = trace + traces[comm[3]] = trace{ + numKernelFrames: len(ebpfTrace.KernelFrames), + frames: libpf.EbpfFrame(slices.Clone(ebpfTrace.FrameData)), + } } } + nativeFrame := libpf.NewEbpfFrame(libpf.NativeFrame, 0, 2, 21) + nativeFrame[1] = 1337 + tests := map[string]struct { // id identifies the trace to inspect (encoded in COMM[3]). id uint8 @@ -157,51 +144,16 @@ Loop: hasKernelFrames bool // userSpaceTrace holds a single Trace with just the user-space portion of the trace // that will be verified against the returned Trace. - userSpaceTrace host.Trace + userSpaceTrace libpf.EbpfFrame }{ "Single Native Frame": { - id: 1, - userSpaceTrace: host.Trace{ - Frames: []host.Frame{{ - File: 1337, - Lineno: 21, - Type: support.FrameMarkerNative, - }}, - }, + id: 1, + userSpaceTrace: nativeFrame, }, "Single Native Frame with Kernel Frames": { id: 2, hasKernelFrames: true, - userSpaceTrace: host.Trace{ - Frames: []host.Frame{{ - File: 1337, - Lineno: 21, - Type: support.FrameMarkerNative, - }}, - }, - }, - "Three Python Frames": { - id: 3, - userSpaceTrace: host.Trace{ - Frames: []host.Frame{{ - File: 1337, - Lineno: 42, - Type: support.FrameMarkerNative, - }, { - File: 1338, - Lineno: 21, - Type: support.FrameMarkerNative, - }, { - File: 1339, - Lineno: 22, - Type: support.FrameMarkerPython, - }}, - }, - }, - "Maximum Length Trace": { - id: 4, - hasKernelFrames: true, - userSpaceTrace: generateMaxLengthTrace(), + userSpaceTrace: nativeFrame, }, } @@ -211,10 +163,8 @@ Loop: trace, ok := traces[testcase.id] require.Truef(t, ok, "trace ID %d not received", testcase.id) - numKernelFrames := len(trace.KernelFrames) - userspaceFrameCount := len(trace.Frames) + numKernelFrames := trace.numKernelFrames - assert.Equal(t, len(testcase.userSpaceTrace.Frames), userspaceFrameCount) assert.False(t, !testcase.hasKernelFrames && numKernelFrames > 0, "unexpected kernel frames") @@ -227,10 +177,9 @@ Loop: assert.Falsef(t, testcase.hasKernelFrames && numKernelFrames < 2, "expected at least 2 kernel frames, but got %d", numKernelFrames) - t.Logf("Received %d user frames and %d kernel frames", - userspaceFrameCount, numKernelFrames) - - validateTrace(t, &testcase.userSpaceTrace, trace) + t.Logf("Received %d framedata and %d kernel frames", + len(trace.frames), numKernelFrames) + assert.Equal(t, testcase.userSpaceTrace, trace.frames) }) } } diff --git a/tracer/events.go b/tracer/events.go index 3ecc05205..87a8394f3 100644 --- a/tracer/events.go +++ b/tracer/events.go @@ -15,7 +15,7 @@ import ( "github.com/cilium/ebpf/perf" "go.opentelemetry.io/ebpf-profiler/internal/log" - "go.opentelemetry.io/ebpf-profiler/host" + "go.opentelemetry.io/ebpf-profiler/libpf" "go.opentelemetry.io/ebpf-profiler/metrics" "go.opentelemetry.io/ebpf-profiler/process" "go.opentelemetry.io/ebpf-profiler/support" @@ -135,7 +135,7 @@ func startPerfEventMonitor(ctx context.Context, perfEventMap *ebpf.Map, // Returns a function that can be called to retrieve perf event array // error counts. func (t *Tracer) startTraceEventMonitor(ctx context.Context, - traceOutChan chan<- *host.Trace, + traceOutChan chan<- *libpf.EbpfTrace, ) func() []metrics.Metric { eventsMap := t.ebpfMaps["trace_events"] eventReader, err := perf.NewReader(eventsMap, @@ -152,7 +152,7 @@ func (t *Tracer) startTraceEventMonitor(ctx context.Context, var lostEventsCount, readErrorCount, noDataCount atomic.Uint64 go func() { var data perf.Record - var oldKTime, minKTime times.KTime + var oldKTime, minKTime int64 var eventCount int pollTicker := time.NewTicker(t.intervals.TracePollInterval()) @@ -251,10 +251,10 @@ func (t *Tracer) startTraceEventMonitor(ctx context.Context, if minKTime > 0 && minKTime <= oldKTime { // If minKTime is smaller than oldKTime, use it and reset it // to avoid a repeat during next iteration. - t.processManager.ProcessedUntil(minKTime) + t.processManager.ProcessedUntil(times.KTime(minKTime)) minKTime = 0 } else { - t.processManager.ProcessedUntil(oldKTime) + t.processManager.ProcessedUntil(times.KTime(oldKTime)) } } oldKTime = minKTime diff --git a/tracer/tracer.go b/tracer/tracer.go index d30c057bc..f25bec4c4 100644 --- a/tracer/tracer.go +++ b/tracer/tracer.go @@ -13,6 +13,7 @@ import ( "math" "math/rand/v2" "strings" + "sync" "time" "unsafe" @@ -23,7 +24,6 @@ import ( "go.opentelemetry.io/ebpf-profiler/internal/log" "go.opentelemetry.io/ebpf-profiler/libpf/pfunsafe" - "go.opentelemetry.io/ebpf-profiler/host" "go.opentelemetry.io/ebpf-profiler/kallsyms" "go.opentelemetry.io/ebpf-profiler/libpf" "go.opentelemetry.io/ebpf-profiler/libpf/xsync" @@ -90,6 +90,9 @@ type Tracer struct { // associated eBPF maps. processManager *pm.ProcessManager + // tracePool is cache of libpf.EbpfTrace to avoid GC pressure + tracePool sync.Pool + // triggerPIDProcessing is used as manual trigger channel to request immediate // processing of pending PIDs. This is requested on notifications from eBPF code // when process events take place (new, exit, unknown PC). @@ -196,6 +199,14 @@ func schedProcessFreeHookName(progNames libpf.Set[string]) string { return schedProcessFreeV2 } +func newTracePool() sync.Pool { + return sync.Pool{ + New: func() any { + return &libpf.EbpfTrace{} + }, + } +} + // NewTracer loads eBPF code and map definitions from the ELF module at the configured path. func NewTracer(ctx context.Context, cfg *Config) (*Tracer, error) { kernelSymbolizer, err := kallsyms.NewSymbolizer() @@ -235,6 +246,7 @@ func NewTracer(ctx context.Context, cfg *Config) (*Tracer, error) { kernelSymbolizer: kernelSymbolizer, processManager: processManager, triggerPIDProcessing: make(chan bool, 1), + tracePool: newTracePool(), pidEvents: make(chan libpf.PIDTID, pidEventBufferSize), ebpfMaps: ebpfMaps, ebpfProgs: ebpfProgs, @@ -739,7 +751,7 @@ func loadProgram(ebpfProgs map[string]*cebpf.Program, tailcallMap *cebpf.Map, // readKernelFrames fetches the kernel stack frames for a particular kstackID and // returns them as symbolized libpf.Frames. -func (t *Tracer) readKernelFrames(kstackID int32) (libpf.Frames, error) { +func (t *Tracer) readKernelFrames(kstackID int32, oldFrames libpf.Frames) (libpf.Frames, error) { cKstackID := kstackID kstackVal := make([]uint64, support.PerfMaxStackDepth) @@ -756,7 +768,10 @@ func (t *Tracer) readKernelFrames(kstackID int32) (libpf.Frames, error) { kstackLen++ } - frames := make(libpf.Frames, 0, kstackLen) + frames := oldFrames + if kstackLen > uint32(len(frames)) { + frames = make(libpf.Frames, 0, kstackLen) + } for i := uint32(0); i < kstackLen; i++ { address := libpf.Address(kstackVal[i]) frame := libpf.Frame{ @@ -911,24 +926,25 @@ func (t *Tracer) eBPFMetricsCollector( // // If the raw trace contains a kernel stack ID, the kernel stack is also // retrieved and inserted at the appropriate position. -func (t *Tracer) loadBpfTrace(raw []byte, cpu int) *host.Trace { - frameListOffs := int(unsafe.Offsetof(support.Trace{}.Frames)) +func (t *Tracer) loadBpfTrace(raw []byte, cpu int) *libpf.EbpfTrace { + frameListOffs := int(unsafe.Offsetof(support.Trace{}.Frame_data)) if len(raw) < frameListOffs { panic("trace record too small") } - frameSize := support.Sizeof_Frame ptr := traceFromRaw(raw) + frameDataLen := int(ptr.Frame_data_len) * 8 // NOTE: can't do exact check here: kernel adds a few padding bytes to messages. - if len(raw) < frameListOffs+int(ptr.Stack_len)*frameSize { + if len(raw) < frameListOffs+frameDataLen { panic("unexpected record size") } pid := libpf.PID(ptr.Pid) procMeta := t.processManager.MetaForPID(pid) - trace := &host.Trace{ + trace := t.tracePool.Get().(*libpf.EbpfTrace) + *trace = libpf.EbpfTrace{ Comm: goString(ptr.Comm[:]), ExecutablePath: procMeta.Executable, ContainerID: procMeta.ContainerID, @@ -939,7 +955,7 @@ func (t *Tracer) loadBpfTrace(raw []byte, cpu int) *host.Trace { TID: libpf.PID(ptr.Tid), Origin: libpf.Origin(ptr.Origin), OffTime: int64(ptr.Offtime), - KTime: times.KTime(ptr.Ktime), + KTime: int64(ptr.Ktime), CPU: cpu, EnvVars: procMeta.EnvVariables, } @@ -955,7 +971,7 @@ func (t *Tracer) loadBpfTrace(raw []byte, cpu int) *host.Trace { if ptr.Kernel_stack_id >= 0 { var err error - trace.KernelFrames, err = t.readKernelFrames(ptr.Kernel_stack_id) + trace.KernelFrames, err = t.readKernelFrames(ptr.Kernel_stack_id, trace.KernelFrames) if err != nil { log.Errorf("Failed to get kernel stack frames: %v", err) } @@ -971,22 +987,16 @@ func (t *Tracer) loadBpfTrace(raw []byte, cpu int) *host.Trace { } } - trace.Frames = make([]host.Frame, ptr.Stack_len) - for i := 0; i < int(ptr.Stack_len); i++ { - rawFrame := &ptr.Frames[i] - trace.Frames[i] = host.Frame{ - File: host.FileID(rawFrame.File_id), - Lineno: libpf.AddressOrLineno(rawFrame.Addr_or_line), - Type: libpf.FrameType(rawFrame.Kind), - ReturnAddress: rawFrame.Return_address != 0, - } - } + trace.NumFrames = int(ptr.Num_frames) + trace.FrameData = trace.FrameDataBuf[:ptr.Frame_data_len] + copy(trace.FrameData, ptr.Frame_data[:ptr.Frame_data_len]) + return trace } // StartMapMonitors starts goroutines for collecting metrics and monitoring eBPF // maps for tracepoints, new traces, trace count updates and unknown PCs. -func (t *Tracer) StartMapMonitors(ctx context.Context, traceOutChan chan<- *host.Trace) error { +func (t *Tracer) StartMapMonitors(ctx context.Context, traceOutChan chan<- *libpf.EbpfTrace) error { if err := t.kernelSymbolizer.StartMonitor(ctx); err != nil { log.Warnf("Failed to start kallsyms monitor: %v", err) } @@ -1208,6 +1218,10 @@ func (t *Tracer) AttachProbes(probes []string) error { return nil } -func (t *Tracer) HandleTrace(bpfTrace *host.Trace) { +func (t *Tracer) HandleTrace(bpfTrace *libpf.EbpfTrace) { t.processManager.HandleTrace(bpfTrace) + + // Reclain the EbpfTrace + bpfTrace.KernelFrames = bpfTrace.KernelFrames[0:0] + t.tracePool.Put(bpfTrace) }