Skip to content

fix(python): handle cold interpreter func ranges #414

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
korniltsev wants to merge 4 commits into from
Closed

fix(python): handle cold interpreter func ranges #414

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
85861b1
fix(python): handle cold interpreter func chunks
korniltsev Mar 21, 2025
1346f70
Lint
korniltsev Mar 21, 2025
71ce508
update arm kernel blobs
korniltsev Mar 21, 2025
43b26dc
revert debug-file-lookup changes
korniltsev Mar 24, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions interpreter/loaderinfo.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,3 +68,7 @@ func (i *LoaderInfo) FileName() string {
func (i *LoaderInfo) Gaps() []util.Range {
return i.gaps
}

func (i *LoaderInfo) ElfOpener() pfelf.ELFOpener {
return i.elfRef.ELFOpener
}
57 changes: 45 additions & 12 deletions interpreter/python/python.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -750,19 +750,9 @@ func Loader(ebpf interpreter.EbpfHandler, info *interpreter.LoaderInfo) (interpr
autoTLSKey += 4
}

// The Python main interpreter loop history in CPython git is:
//
//nolint:lll
// 87af12bff33 v3.11 2022-02-15 _PyEval_EvalFrameDefault(PyThreadState*,_PyInterpreterFrame*,int)
// ae0a2b75625 v3.10 2021-06-25 _PyEval_EvalFrameDefault(PyThreadState*,_interpreter_frame*,int)
// 0b72b23fb0c v3.9 2020-03-12 _PyEval_EvalFrameDefault(PyThreadState*,PyFrameObject*,int)
// 3cebf938727 v3.6 2016-09-05 _PyEval_EvalFrameDefault(PyFrameObject*,int)
// 49fd7fa4431 v3.0 2006-04-21 PyEval_EvalFrameEx(PyFrameObject*,int)
interpRanges, err := info.GetSymbolAsRanges("_PyEval_EvalFrameDefault")
interpRanges, err := findInterpreterRanges(ef, info)
if err != nil {
if interpRanges, err = info.GetSymbolAsRanges("PyEval_EvalFrameEx"); err != nil {
return nil, err
}
return nil, err
}

pd := &pythonData{
Expand Down Expand Up @@ -837,3 +827,46 @@ func Loader(ebpf interpreter.EbpfHandler, info *interpreter.LoaderInfo) (interpr

return pd, nil
}

func findInterpreterRanges(ef *pfelf.File, info *interpreter.LoaderInfo) ([]util.Range, error) {
// The Python main interpreter loop history in CPython git is:
//
//nolint:lll
// 87af12bff33 v3.11 2022-02-15 _PyEval_EvalFrameDefault(PyThreadState*,_PyInterpreterFrame*,int)
// ae0a2b75625 v3.10 2021-06-25 _PyEval_EvalFrameDefault(PyThreadState*,_interpreter_frame*,int)
// 0b72b23fb0c v3.9 2020-03-12 _PyEval_EvalFrameDefault(PyThreadState*,PyFrameObject*,int)
// 3cebf938727 v3.6 2016-09-05 _PyEval_EvalFrameDefault(PyFrameObject*,int)
// 49fd7fa4431 v3.0 2006-04-21 PyEval_EvalFrameEx(PyFrameObject*,int)
interpRanges, err := info.GetSymbolAsRanges("_PyEval_EvalFrameDefault")
if err != nil {
if interpRanges, err = info.GetSymbolAsRanges("PyEval_EvalFrameEx"); err != nil {
return nil, err
}
return interpRanges, nil
}
if len(interpRanges) == 0 {
return nil, errors.New("no _PyEval_EvalFrameDefault/PyEval_EvalFrameEx symbol found")
}
gnuBuildID, _ := ef.GetBuildID()
if coldRange, ok := interpreterColdRanges[gnuBuildID]; ok {
interpRanges = append(interpRanges, coldRange)
return interpRanges, nil
}
// TODO: find cold ranges for unknown binaries (not in interpreterColdRanges)
// see tools/coredump/testdata/amd64/alpine320-nobuildid.json
// https://github.com/open-telemetry/opentelemetry-ebpf-profiler/issues/416
return interpRanges, nil
}

var interpreterColdRanges = map[string]util.Range{
// alpine:3.20
"4913fe1380aebd0f4f0d69411b797d7e22d2799b": {
Start: 0x88a9a,
End: 0x88a9a + 0xdcdf,
},
// alpine 3.21
"f0f26a21d40d3c089975a8b136fc2469df40a0e6": {
Start: 0x89228,
End: 0x89228 + 0x35d9,
},
}
49 changes: 31 additions & 18 deletions processmanager/ebpf/ebpf.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -253,29 +253,42 @@ func LoadMaps(ctx context.Context, maps map[string]*cebpf.Map) (EbpfHandler, err
// UpdateInterpreterOffsets adds the given moduleRanges to the eBPF map interpreterOffsets.
func (impl *ebpfMapsImpl) UpdateInterpreterOffsets(ebpfProgIndex uint16, fileID host.FileID,
offsetRanges []util.Range) error {
if offsetRanges == nil {
return errors.New("offsetRanges is nil")
}
for _, offsetRange := range offsetRanges {
// The keys of this map are executable-id-and-offset-into-text entries, and
// the offset_range associated with them gives the precise area in that page
// where the main interpreter loop is located. This is required to unwind
// nicely from native code into interpreted code.
key := uint64(fileID)
value := C.OffsetRange{
lower_offset: C.u64(offsetRange.Start),
upper_offset: C.u64(offsetRange.End),
program_index: C.u16(ebpfProgIndex),
}
if err := impl.interpreterOffsets.Update(unsafe.Pointer(&key), unsafe.Pointer(&value),
cebpf.UpdateAny); err != nil {
log.Fatalf("Failed to place interpreter range in map: %v", err)
}
key, value, err := InterpreterOffsetKeyValue(ebpfProgIndex, fileID, offsetRanges)
if err != nil {
return err
}
if err := impl.interpreterOffsets.Update(unsafe.Pointer(&key), unsafe.Pointer(&value),
cebpf.UpdateAny); err != nil {
log.Fatalf("Failed to place interpreter range in map: %v", err)
}

return nil
}

func InterpreterOffsetKeyValue(ebpfProgIndex uint16, fileID host.FileID,
offsetRanges []util.Range) (key uint64, value C.OffsetRange, err error) {
if len(offsetRanges) != 1 && len(offsetRanges) != 2 {
return 0, C.OffsetRange{}, fmt.Errorf("ivalid ranges %+v", offsetRanges)
}
// The keys of this map are executable-id-and-offset-into-text entries, and
// the offset_range associated with them gives the precise area in that page
// where the main interpreter loop is located. This is required to unwind
// nicely from native code into interpreted code.
key = uint64(fileID)
first := offsetRanges[0]
value = C.OffsetRange{
lower_offset1: C.u64(first.Start),
upper_offset1: C.u64(first.End),
program_index: C.u16(ebpfProgIndex),
}
if len(offsetRanges) == 2 {
second := offsetRanges[1]
value.lower_offset2 = C.u64(second.Start)
value.upper_offset2 = C.u64(second.End)
}
return key, value, nil
}

// getInterpreterTypeMap returns the eBPF map for the given typ
// or an error if typ is not supported.
func (impl *ebpfMapsImpl) getInterpreterTypeMap(typ libpf.InterpreterType) (*cebpf.Map, error) {
Expand Down
2 changes: 1 addition & 1 deletion support/ebpf/native_stack_trace.ebpf.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ bpf_map_def SEC("maps") interpreter_offsets = {
.type = BPF_MAP_TYPE_HASH,
.key_size = sizeof(u64),
.value_size = sizeof(OffsetRange),
.max_entries = 32,
.max_entries = 256,
};

// Maps fileID and page to information of stack deltas associated with that page.
Expand Down
4 changes: 3 additions & 1 deletion support/ebpf/tracemgmt.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -422,7 +422,9 @@ static inline int get_next_interpreter(PerCPURecord *record)
// Check if the section id happens to be in the interpreter map.
OffsetRange *range = bpf_map_lookup_elem(&interpreter_offsets, &section_id);
if (range != 0) {
if ((section_offset >= range->lower_offset) && (section_offset <= range->upper_offset)) {
if (
((section_offset >= range->lower_offset1) && (section_offset <= range->upper_offset1)) ||
((section_offset >= range->lower_offset2) && (section_offset <= range->upper_offset2))) {
DEBUG_PRINT("interpreter_offsets match %d", range->program_index);
if (!unwinder_is_done(record, range->program_index)) {
increment_metric(metricID_UnwindCallInterpreter);
Expand Down
Binary file modified support/ebpf/tracer.ebpf.release.amd64
View file
Open in desktop
Binary file not shown.
Binary file modified support/ebpf/tracer.ebpf.release.arm64
View file
Open in desktop
Binary file not shown.
7 changes: 5 additions & 2 deletions support/ebpf/types.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -773,8 +773,11 @@ typedef struct StackDeltaPageInfo {
// the upper boundary of the loop, and the relevant index to call in the prog
// array.
typedef struct OffsetRange {
u64 lower_offset;
u64 upper_offset;
u64 lower_offset1;
u64 upper_offset1;
// extra range for .cold interpreter chunk
u64 lower_offset2;
u64 upper_offset2;
u16 program_index; // The interpreter-specific program index to call.
} OffsetRange;

Expand Down
7 changes: 7 additions & 0 deletions tools/coredump/coredump_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,10 @@ import (
)

func TestCoreDumps(t *testing.T) {
var skip = map[string]bool{
// https://github.com/open-telemetry/opentelemetry-ebpf-profiler/issues/416
"testdata/amd64/alpine320-nobuildid.json": true,
}
cases, err := findTestCases(true)
require.NoError(t, err)
require.NotEmpty(t, cases)
Expand All @@ -19,6 +23,9 @@ func TestCoreDumps(t *testing.T) {
require.NoError(t, err)

for _, filename := range cases {
if skip[filename] {
continue
}
filename := filename
t.Run(filename, func(t *testing.T) {
testCase, err := readTestCase(filename)
Expand Down
10 changes: 4 additions & 6 deletions tools/coredump/ebpfmaps.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,13 +44,11 @@ func (emc *ebpfMapsCoredump) CollectMetrics() []metrics.Metric {

func (emc *ebpfMapsCoredump) UpdateInterpreterOffsets(ebpfProgIndex uint16,
fileID host.FileID, offsetRanges []util.Range) error {
offsetRange := offsetRanges[0]
value := C.OffsetRange{
lower_offset: C.u64(offsetRange.Start),
upper_offset: C.u64(offsetRange.End),
program_index: C.u16(ebpfProgIndex),
key, value, err := pmebpf.InterpreterOffsetKeyValue(ebpfProgIndex, fileID, offsetRanges)
if err != nil {
return err
}
emc.ctx.addMap(&C.interpreter_offsets, C.u64(fileID), libpf.SliceFrom(&value))
emc.ctx.addMap(&C.interpreter_offsets, C.u64(key), libpf.SliceFrom(&value))
return nil
}

Expand Down
87 changes: 87 additions & 0 deletions tools/coredump/testdata/amd64/alpine320-nobuildid.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
{
"coredump-ref": "3eb6bae4e0089983f436d6bbd4a0b7ee0d72738eac29f15495494f53bc82263d",
"threads": [
{
"lwp": 80,
"frames": [
"fib+1 in /mnt/trash/qwe.py:2",
"fib+3 in /mnt/trash/qwe.py:4",
"fib+3 in /mnt/trash/qwe.py:4",
"fib+3 in /mnt/trash/qwe.py:4",
"fib+3 in /mnt/trash/qwe.py:4",
"fib+3 in /mnt/trash/qwe.py:4",
"fib+3 in /mnt/trash/qwe.py:4",
"fib+3 in /mnt/trash/qwe.py:4",
"fib+3 in /mnt/trash/qwe.py:4",
"fib+3 in /mnt/trash/qwe.py:4",
"fib+3 in /mnt/trash/qwe.py:4",
"fib+3 in /mnt/trash/qwe.py:4",
"fib+3 in /mnt/trash/qwe.py:4",
"fib+3 in /mnt/trash/qwe.py:4",
"fib+3 in /mnt/trash/qwe.py:4",
"fib+3 in /mnt/trash/qwe.py:4",
"fib+3 in /mnt/trash/qwe.py:4",
"fib+3 in /mnt/trash/qwe.py:4",
"fib+3 in /mnt/trash/qwe.py:4",
"fib+3 in /mnt/trash/qwe.py:4",
"fib+3 in /mnt/trash/qwe.py:4",
"fib+3 in /mnt/trash/qwe.py:4",
"fib+3 in /mnt/trash/qwe.py:4",
"fib+3 in /mnt/trash/qwe.py:4",
"fib+3 in /mnt/trash/qwe.py:4",
"fib+3 in /mnt/trash/qwe.py:4",
"fib+3 in /mnt/trash/qwe.py:4",
"fib+3 in /mnt/trash/qwe.py:4",
"fib+3 in /mnt/trash/qwe.py:4",
"fib+3 in /mnt/trash/qwe.py:4",
"fib+3 in /mnt/trash/qwe.py:4",
"fib+3 in /mnt/trash/qwe.py:4",
"fib+3 in /mnt/trash/qwe.py:4",
"fib+3 in /mnt/trash/qwe.py:4",
"fib+3 in /mnt/trash/qwe.py:4",
"fib+3 in /mnt/trash/qwe.py:4",
"fib+3 in /mnt/trash/qwe.py:4",
"<module>+6 in /mnt/trash/qwe.py:7",
"<interpreter trampoline>+0 in <shim>:1",
"libpython3.12.so.1.0+0x95b58",
"libpython3.12.so.1.0+0x1ecfe2",
"libpython3.12.so.1.0+0x2136b6",
"libpython3.12.so.1.0+0x20c91b",
"libpython3.12.so.1.0+0x226f72",
"libpython3.12.so.1.0+0x2264f1",
"libpython3.12.so.1.0+0x2260d3",
"libpython3.12.so.1.0+0x21f742",
"libpython3.12.so.1.0+0x1d6ed6",
"ld-musl-x86_64.so.1+0x1c709",
"python3.12+0x1045",
"<unwinding aborted due to error native_small_pc>"
]
}
],
"modules": [
{
"ref": "983f4cac5caf833fbf7d5d28ac8d6a55d3cbab6152d37a246af1a9991f72d8b1",
"local-path": "/usr/lib/debug/lib/ld-musl-x86_64.so.1.debug"
},
{
"ref": "497dd0d2b4a80bfd11339306c84aa752d811f612a398cb526a0a9ac2f426c0b8",
"local-path": "/usr/lib/libpython3.12.so.1.0"
},
{
"ref": "02a162bd137903ef2511ccb6edc32eaa63a9c9c66c8474f6ce7d9d47fc5a1e71",
"local-path": "/usr/lib/debug/usr/lib/libpython3.12.so.1.0.debug"
},
{
"ref": "9cf22fb673cad95247584fd0bc21b95aa37ae152a8bd01022a494e4f7ec3854c",
"local-path": "/usr/bin/python3.12"
},
{
"ref": "f2c83c6e5aea0e1e42b69e344a2eb0aac35b361cad7497157f727dbe30b8565e",
"local-path": "/usr/lib/debug/usr/bin/python3.12.debug"
},
{
"ref": "c9cbfe6a266c104f74629a257e2017020cba7a0da97caefa4c1d068ea6fe698c",
"local-path": "/lib/ld-musl-x86_64.so.1"
}
]
}
85 changes: 85 additions & 0 deletions tools/coredump/testdata/amd64/alpine320.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
{
"coredump-ref": "4652115623df00987a1a480431a360edbd67b0795e9529543d40be190e37c74d",
"threads": [
{
"lwp": 10,
"frames": [
"libpython3.12.so.1.0+0x19b80d",
"fib+3 in //mnt/trash/qwe.py:4",
"fib+3 in //mnt/trash/qwe.py:4",
"fib+3 in //mnt/trash/qwe.py:4",
"fib+3 in //mnt/trash/qwe.py:4",
"fib+3 in //mnt/trash/qwe.py:4",
"fib+3 in //mnt/trash/qwe.py:4",
"fib+3 in //mnt/trash/qwe.py:4",
"fib+3 in //mnt/trash/qwe.py:4",
"fib+3 in //mnt/trash/qwe.py:4",
"fib+3 in //mnt/trash/qwe.py:4",
"fib+3 in //mnt/trash/qwe.py:4",
"fib+3 in //mnt/trash/qwe.py:4",
"fib+3 in //mnt/trash/qwe.py:4",
"fib+3 in //mnt/trash/qwe.py:4",
"fib+3 in //mnt/trash/qwe.py:4",
"fib+3 in //mnt/trash/qwe.py:4",
"fib+3 in //mnt/trash/qwe.py:4",
"fib+3 in //mnt/trash/qwe.py:4",
"fib+3 in //mnt/trash/qwe.py:4",
"fib+3 in //mnt/trash/qwe.py:4",
"fib+3 in //mnt/trash/qwe.py:4",
"fib+3 in //mnt/trash/qwe.py:4",
"fib+3 in //mnt/trash/qwe.py:4",
"fib+3 in //mnt/trash/qwe.py:4",
"fib+3 in //mnt/trash/qwe.py:4",
"fib+3 in //mnt/trash/qwe.py:4",
"fib+3 in //mnt/trash/qwe.py:4",
"fib+3 in //mnt/trash/qwe.py:4",
"fib+3 in //mnt/trash/qwe.py:4",
"fib+3 in //mnt/trash/qwe.py:4",
"fib+3 in //mnt/trash/qwe.py:4",
"fib+3 in //mnt/trash/qwe.py:4",
"fib+3 in //mnt/trash/qwe.py:4",
"fib+3 in //mnt/trash/qwe.py:4",
"<module>+6 in //mnt/trash/qwe.py:7",
"<interpreter trampoline>+0 in <shim>:1",
"libpython3.12.so.1.0+0x91617",
"libpython3.12.so.1.0+0x1ecfe2",
"libpython3.12.so.1.0+0x2136b6",
"libpython3.12.so.1.0+0x20c91b",
"libpython3.12.so.1.0+0x226f72",
"libpython3.12.so.1.0+0x2264f1",
"libpython3.12.so.1.0+0x2260d3",
"libpython3.12.so.1.0+0x21f742",
"libpython3.12.so.1.0+0x1d6ed6",
"ld-musl-x86_64.so.1+0x1c709",
"python3.12+0x1045",
"<unwinding aborted due to error native_small_pc>"
]
}
],
"modules": [
{
"ref": "a621ea443bba20ffbdb5322633c5a1bf439905baa5822973b2cafc4106c64789",
"local-path": "/usr/lib/libpython3.12.so.1.0"
},
{
"ref": "9ef79f767e2c2cfde043a023acb0021f6261f560b27035c961d282cecc026db9",
"local-path": "/usr/lib/debug/usr/lib/libpython3.12.so.1.0.debug"
},
{
"ref": "c9cbfe6a266c104f74629a257e2017020cba7a0da97caefa4c1d068ea6fe698c",
"local-path": "/lib/ld-musl-x86_64.so.1"
},
{
"ref": "983f4cac5caf833fbf7d5d28ac8d6a55d3cbab6152d37a246af1a9991f72d8b1",
"local-path": "/usr/lib/debug/lib/ld-musl-x86_64.so.1.debug"
},
{
"ref": "9cf22fb673cad95247584fd0bc21b95aa37ae152a8bd01022a494e4f7ec3854c",
"local-path": "/usr/bin/python3.12"
},
{
"ref": "f2c83c6e5aea0e1e42b69e344a2eb0aac35b361cad7497157f727dbe30b8565e",
"local-path": "/usr/lib/debug/usr/bin/python3.12.debug"
}
]
}