From 7b3509f3ca6e383bb59e1ed315ca726abfb6a1cf Mon Sep 17 00:00:00 2001 From: Nikola Grcevski Date: Mon, 4 Mar 2024 15:37:06 -0500 Subject: [PATCH 01/14] WIP: Switch writing HTTP Traceparent without external memory --- internal/pkg/inject/offset_results.json | 331 +++++++++++++++++- .../bpf/net/http/client/bpf/probe.bpf.c | 76 +++- .../bpf/net/http/client/bpf_bpfel_arm64.go | 6 + .../bpf/net/http/client/bpf_bpfel_x86.go | 6 + .../net/http/client/bpf_no_tp_bpfel_arm64.go | 169 +++++++++ .../net/http/client/bpf_no_tp_bpfel_x86.go | 169 +++++++++ .../bpf/net/http/client/probe.go | 31 +- internal/tools/inspect/cmd/offsetgen/main.go | 2 + .../cmd/offsetgen/{verions.go => versions.go} | 0 9 files changed, 786 insertions(+), 4 deletions(-) create mode 100644 internal/pkg/instrumentation/bpf/net/http/client/bpf_no_tp_bpfel_arm64.go create mode 100644 internal/pkg/instrumentation/bpf/net/http/client/bpf_no_tp_bpfel_x86.go rename internal/tools/inspect/cmd/offsetgen/{verions.go => versions.go} (100%) diff --git a/internal/pkg/inject/offset_results.json b/internal/pkg/inject/offset_results.json index 0657f3991..138484fe2 100644 --- a/internal/pkg/inject/offset_results.json +++ b/internal/pkg/inject/offset_results.json @@ -130,7 +130,8 @@ "0.18.0", "0.19.0", "0.20.0", - "0.21.0" + "0.21.0", + "0.22.0" ] } ] @@ -166,7 +167,8 @@ "0.18.0", "0.19.0", "0.20.0", - "0.21.0" + "0.21.0", + "0.22.0" ] } ] @@ -3699,6 +3701,331 @@ ] } ] + }, + { + "package": "bufio", + "structs": [ + { + "struct": "Writer", + "fields": [ + { + "field": "buf", + "offsets": [ + { + "offset": 16, + "versions": [ + "1.12.0", + "1.12.1", + "1.12.2", + "1.12.3", + "1.12.4", + "1.12.5", + "1.12.6", + "1.12.7", + "1.12.8", + "1.12.9", + "1.12.10", + "1.12.11", + "1.12.12", + "1.12.13", + "1.12.14", + "1.12.15", + "1.12.16", + "1.12.17", + "1.13.0", + "1.13.1", + "1.13.2", + "1.13.3", + "1.13.4", + "1.13.5", + "1.13.6", + "1.13.7", + "1.13.8", + "1.13.9", + "1.13.10", + "1.13.11", + "1.13.12", + "1.13.13", + "1.13.14", + "1.13.15", + "1.14.0", + "1.14.1", + "1.14.2", + "1.14.3", + "1.14.4", + "1.14.5", + "1.14.6", + "1.14.7", + "1.14.8", + "1.14.9", + "1.14.10", + "1.14.11", + "1.14.12", + "1.14.13", + "1.14.14", + "1.14.15", + "1.15.0", + "1.15.1", + "1.15.2", + "1.15.3", + "1.15.4", + "1.15.5", + "1.15.6", + "1.15.7", + "1.15.8", + "1.15.9", + "1.15.10", + "1.15.11", + "1.15.12", + "1.15.13", + "1.15.14", + "1.15.15", + "1.16.0", + "1.16.1", + "1.16.2", + "1.16.3", + "1.16.4", + "1.16.5", + "1.16.6", + "1.16.7", + "1.16.8", + "1.16.9", + "1.16.10", + "1.16.11", + "1.16.12", + "1.16.13", + "1.16.14", + "1.16.15", + "1.17.0", + "1.17.1", + "1.17.2", + "1.17.3", + "1.17.4", + "1.17.5", + "1.17.6", + "1.17.7", + "1.17.8", + "1.17.9", + "1.17.10", + "1.17.11", + "1.17.12", + "1.17.13", + "1.18.0", + "1.18.1", + "1.18.2", + "1.18.3", + "1.18.4", + "1.18.5", + "1.18.6", + "1.18.7", + "1.18.8", + "1.18.9", + "1.18.10", + "1.19.0", + "1.19.1", + "1.19.2", + "1.19.3", + "1.19.4", + "1.19.5", + "1.19.6", + "1.19.7", + "1.19.8", + "1.19.9", + "1.19.10", + "1.19.11", + "1.19.12", + "1.19.13", + "1.20.0", + "1.20.1", + "1.20.2", + "1.20.3", + "1.20.4", + "1.20.5", + "1.20.6", + "1.20.7", + "1.20.8", + "1.20.9", + "1.20.10", + "1.20.11", + "1.20.12", + "1.20.13", + "1.20.14", + "1.21.0", + "1.21.1", + "1.21.2", + "1.21.3", + "1.21.4", + "1.21.5", + "1.21.6", + "1.21.7", + "1.22.0" + ] + } + ] + } + ] + }, + { + "struct": "Writer", + "fields": [ + { + "field": "n", + "offsets": [ + { + "offset": 40, + "versions": [ + "1.12.0", + "1.12.1", + "1.12.2", + "1.12.3", + "1.12.4", + "1.12.5", + "1.12.6", + "1.12.7", + "1.12.8", + "1.12.9", + "1.12.10", + "1.12.11", + "1.12.12", + "1.12.13", + "1.12.14", + "1.12.15", + "1.12.16", + "1.12.17", + "1.13.0", + "1.13.1", + "1.13.2", + "1.13.3", + "1.13.4", + "1.13.5", + "1.13.6", + "1.13.7", + "1.13.8", + "1.13.9", + "1.13.10", + "1.13.11", + "1.13.12", + "1.13.13", + "1.13.14", + "1.13.15", + "1.14.0", + "1.14.1", + "1.14.2", + "1.14.3", + "1.14.4", + "1.14.5", + "1.14.6", + "1.14.7", + "1.14.8", + "1.14.9", + "1.14.10", + "1.14.11", + "1.14.12", + "1.14.13", + "1.14.14", + "1.14.15", + "1.15.0", + "1.15.1", + "1.15.2", + "1.15.3", + "1.15.4", + "1.15.5", + "1.15.6", + "1.15.7", + "1.15.8", + "1.15.9", + "1.15.10", + "1.15.11", + "1.15.12", + "1.15.13", + "1.15.14", + "1.15.15", + "1.16.0", + "1.16.1", + "1.16.2", + "1.16.3", + "1.16.4", + "1.16.5", + "1.16.6", + "1.16.7", + "1.16.8", + "1.16.9", + "1.16.10", + "1.16.11", + "1.16.12", + "1.16.13", + "1.16.14", + "1.16.15", + "1.17.0", + "1.17.1", + "1.17.2", + "1.17.3", + "1.17.4", + "1.17.5", + "1.17.6", + "1.17.7", + "1.17.8", + "1.17.9", + "1.17.10", + "1.17.11", + "1.17.12", + "1.17.13", + "1.18.0", + "1.18.1", + "1.18.2", + "1.18.3", + "1.18.4", + "1.18.5", + "1.18.6", + "1.18.7", + "1.18.8", + "1.18.9", + "1.18.10", + "1.19.0", + "1.19.1", + "1.19.2", + "1.19.3", + "1.19.4", + "1.19.5", + "1.19.6", + "1.19.7", + "1.19.8", + "1.19.9", + "1.19.10", + "1.19.11", + "1.19.12", + "1.19.13", + "1.20.0", + "1.20.1", + "1.20.2", + "1.20.3", + "1.20.4", + "1.20.5", + "1.20.6", + "1.20.7", + "1.20.8", + "1.20.9", + "1.20.10", + "1.20.11", + "1.20.12", + "1.20.13", + "1.20.14", + "1.21.0", + "1.21.1", + "1.21.2", + "1.21.3", + "1.21.4", + "1.21.5", + "1.21.6", + "1.21.7", + "1.22.0" + ] + } + ] + } + ] + } + ] } ] } diff --git a/internal/pkg/instrumentation/bpf/net/http/client/bpf/probe.bpf.c b/internal/pkg/instrumentation/bpf/net/http/client/bpf/probe.bpf.c index 18fa211ff..54b88c81a 100644 --- a/internal/pkg/instrumentation/bpf/net/http/client/bpf/probe.bpf.c +++ b/internal/pkg/instrumentation/bpf/net/http/client/bpf/probe.bpf.c @@ -45,6 +45,13 @@ struct __uint(max_entries, 1); } http_client_uprobe_storage_map SEC(".maps"); +struct { + __uint(type, BPF_MAP_TYPE_LRU_HASH); + __type(key, void*); // the headers ptr + __type(value, void*); // request key, goroutine or context ptr + __uint(max_entries, MAX_CONCURRENT); +} http_headers SEC(".maps"); + struct { __uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY); } events SEC(".maps"); @@ -59,7 +66,10 @@ volatile const u64 buckets_ptr_pos; volatile const u64 status_code_pos; volatile const u64 request_host_pos; volatile const u64 request_proto_pos; +volatile const u64 io_writer_buf_ptr_pos; +volatile const u64 io_writer_n_pos; +#ifndef NO_HEADER_PROPAGATION static __always_inline long inject_header(void* headers_ptr, struct span_context* propagated_ctx) { // Read the key-value count - this field must be the first one in the hmap struct as documented in src/runtime/map.go u64 curr_keyvalue_count = 0; @@ -160,6 +170,7 @@ static __always_inline long inject_header(void* headers_ptr, struct span_context bpf_memset((unsigned char *)bucket_map_value, sizeof(struct map_bucket), 0); return 0; } +#endif // This instrumentation attaches uprobe to the following function: // func net/http/transport.roundTrip(req *Request) (*Response, error) @@ -228,11 +239,17 @@ int uprobe_Transport_roundTrip(struct pt_regs *ctx) { // get headers from Request void *headers_ptr = 0; bpf_probe_read(&headers_ptr, sizeof(headers_ptr), (void *)(req_ptr+headers_ptr_pos)); + if (headers_ptr) { + bpf_printk("headers_ptr = %llx, key = %llx", headers_ptr, key); + bpf_map_update_elem(&http_headers, &headers_ptr, &key, 0); + } + +#ifndef NO_HEADER_PROPAGATION long res = inject_header(headers_ptr, &httpReq->sc); if (res < 0) { bpf_printk("uprobe_Transport_roundTrip: Failed to inject header"); } - +#endif // Write event bpf_map_update_elem(&http_events, &key, httpReq, 0); start_tracking_span(context_ptr_val, &httpReq->sc); @@ -267,4 +284,61 @@ int uprobe_Transport_roundTrip_Returns(struct pt_regs *ctx) { bpf_map_delete_elem(&http_events, &key); return 0; +} + +SEC("uprobe/header_writeSubset") +int uprobe_writeSubset(struct pt_regs *ctx) { + u64 headers_pos = 1; + void *headers_ptr = get_argument(ctx, headers_pos); + + u64 io_writer_pos = 3; + void *io_writer_ptr = get_argument(ctx, io_writer_pos); + + bpf_printk("headers_ptr = %llx, io_writer_ptr = %llx", headers_ptr, io_writer_ptr); + + void **key_ptr = bpf_map_lookup_elem(&http_headers, &headers_ptr); + if (key_ptr) { + void *key = *key_ptr; + bpf_printk("Found request key %llx", key); + + struct http_request_t *http_req_span = bpf_map_lookup_elem(&http_events, &key); + if (http_req_span) { + char tp[W3C_VAL_LENGTH]; + span_context_to_w3c_string(&http_req_span->sc, tp); + + bpf_printk("Will write %s", tp); + + void *buf_ptr = 0; + bpf_probe_read(&buf_ptr, sizeof(buf_ptr), (void *)(io_writer_ptr + io_writer_buf_ptr_pos)); // grab buf ptr + if (!buf_ptr) { + goto done; + } + + s64 size = 0; + bpf_probe_read(&size, sizeof(s64), (void *)(io_writer_ptr + io_writer_buf_ptr_pos + 8)); // grab size + + s64 len = 0; + bpf_probe_read(&len, sizeof(s64), (void *)(io_writer_ptr + io_writer_n_pos)); // grab len + + bpf_printk("buf_ptr %llx, len=%d, size=%d", (void*)buf_ptr, len, size); + +#ifndef NO_HEADER_PROPAGATION + if (len < (size - W3C_VAL_LENGTH - W3C_KEY_LENGTH - 4)) { // 4 = strlen(":_") + strlen("\r\n") + char key[W3C_KEY_LENGTH + 2] = "Traceparent: "; + char end[2] = "\r\n"; + bpf_probe_write_user(buf_ptr + (len & 0x0ffff), key, sizeof(key)); + len += W3C_KEY_LENGTH + 2; + bpf_probe_write_user(buf_ptr + (len & 0x0ffff), tp, sizeof(tp)); + len += W3C_VAL_LENGTH; + bpf_probe_write_user(buf_ptr + (len & 0x0ffff), end, sizeof(end)); + len += 2; + bpf_probe_write_user((void *)(io_writer_ptr + io_writer_n_pos), &len, sizeof(len)); + } +#endif + } + } + +done: + bpf_map_delete_elem(&http_headers, &headers_ptr); + return 0; } \ No newline at end of file diff --git a/internal/pkg/instrumentation/bpf/net/http/client/bpf_bpfel_arm64.go b/internal/pkg/instrumentation/bpf/net/http/client/bpf_bpfel_arm64.go index 2811d12d5..71f2da972 100644 --- a/internal/pkg/instrumentation/bpf/net/http/client/bpf_bpfel_arm64.go +++ b/internal/pkg/instrumentation/bpf/net/http/client/bpf_bpfel_arm64.go @@ -75,6 +75,7 @@ type bpfSpecs struct { type bpfProgramSpecs struct { UprobeTransportRoundTrip *ebpf.ProgramSpec `ebpf:"uprobe_Transport_roundTrip"` UprobeTransportRoundTripReturns *ebpf.ProgramSpec `ebpf:"uprobe_Transport_roundTrip_Returns"` + UprobeWriteSubset *ebpf.ProgramSpec `ebpf:"uprobe_writeSubset"` } // bpfMapSpecs contains maps before they are loaded into the kernel. @@ -86,6 +87,7 @@ type bpfMapSpecs struct { GolangMapbucketStorageMap *ebpf.MapSpec `ebpf:"golang_mapbucket_storage_map"` HttpClientUprobeStorageMap *ebpf.MapSpec `ebpf:"http_client_uprobe_storage_map"` HttpEvents *ebpf.MapSpec `ebpf:"http_events"` + HttpHeaders *ebpf.MapSpec `ebpf:"http_headers"` SliceArrayBuffMap *ebpf.MapSpec `ebpf:"slice_array_buff_map"` TrackedSpans *ebpf.MapSpec `ebpf:"tracked_spans"` TrackedSpansBySc *ebpf.MapSpec `ebpf:"tracked_spans_by_sc"` @@ -115,6 +117,7 @@ type bpfMaps struct { GolangMapbucketStorageMap *ebpf.Map `ebpf:"golang_mapbucket_storage_map"` HttpClientUprobeStorageMap *ebpf.Map `ebpf:"http_client_uprobe_storage_map"` HttpEvents *ebpf.Map `ebpf:"http_events"` + HttpHeaders *ebpf.Map `ebpf:"http_headers"` SliceArrayBuffMap *ebpf.Map `ebpf:"slice_array_buff_map"` TrackedSpans *ebpf.Map `ebpf:"tracked_spans"` TrackedSpansBySc *ebpf.Map `ebpf:"tracked_spans_by_sc"` @@ -127,6 +130,7 @@ func (m *bpfMaps) Close() error { m.GolangMapbucketStorageMap, m.HttpClientUprobeStorageMap, m.HttpEvents, + m.HttpHeaders, m.SliceArrayBuffMap, m.TrackedSpans, m.TrackedSpansBySc, @@ -139,12 +143,14 @@ func (m *bpfMaps) Close() error { type bpfPrograms struct { UprobeTransportRoundTrip *ebpf.Program `ebpf:"uprobe_Transport_roundTrip"` UprobeTransportRoundTripReturns *ebpf.Program `ebpf:"uprobe_Transport_roundTrip_Returns"` + UprobeWriteSubset *ebpf.Program `ebpf:"uprobe_writeSubset"` } func (p *bpfPrograms) Close() error { return _BpfClose( p.UprobeTransportRoundTrip, p.UprobeTransportRoundTripReturns, + p.UprobeWriteSubset, ) } diff --git a/internal/pkg/instrumentation/bpf/net/http/client/bpf_bpfel_x86.go b/internal/pkg/instrumentation/bpf/net/http/client/bpf_bpfel_x86.go index 0a4e5275f..e77cf8f80 100644 --- a/internal/pkg/instrumentation/bpf/net/http/client/bpf_bpfel_x86.go +++ b/internal/pkg/instrumentation/bpf/net/http/client/bpf_bpfel_x86.go @@ -75,6 +75,7 @@ type bpfSpecs struct { type bpfProgramSpecs struct { UprobeTransportRoundTrip *ebpf.ProgramSpec `ebpf:"uprobe_Transport_roundTrip"` UprobeTransportRoundTripReturns *ebpf.ProgramSpec `ebpf:"uprobe_Transport_roundTrip_Returns"` + UprobeWriteSubset *ebpf.ProgramSpec `ebpf:"uprobe_writeSubset"` } // bpfMapSpecs contains maps before they are loaded into the kernel. @@ -86,6 +87,7 @@ type bpfMapSpecs struct { GolangMapbucketStorageMap *ebpf.MapSpec `ebpf:"golang_mapbucket_storage_map"` HttpClientUprobeStorageMap *ebpf.MapSpec `ebpf:"http_client_uprobe_storage_map"` HttpEvents *ebpf.MapSpec `ebpf:"http_events"` + HttpHeaders *ebpf.MapSpec `ebpf:"http_headers"` SliceArrayBuffMap *ebpf.MapSpec `ebpf:"slice_array_buff_map"` TrackedSpans *ebpf.MapSpec `ebpf:"tracked_spans"` TrackedSpansBySc *ebpf.MapSpec `ebpf:"tracked_spans_by_sc"` @@ -115,6 +117,7 @@ type bpfMaps struct { GolangMapbucketStorageMap *ebpf.Map `ebpf:"golang_mapbucket_storage_map"` HttpClientUprobeStorageMap *ebpf.Map `ebpf:"http_client_uprobe_storage_map"` HttpEvents *ebpf.Map `ebpf:"http_events"` + HttpHeaders *ebpf.Map `ebpf:"http_headers"` SliceArrayBuffMap *ebpf.Map `ebpf:"slice_array_buff_map"` TrackedSpans *ebpf.Map `ebpf:"tracked_spans"` TrackedSpansBySc *ebpf.Map `ebpf:"tracked_spans_by_sc"` @@ -127,6 +130,7 @@ func (m *bpfMaps) Close() error { m.GolangMapbucketStorageMap, m.HttpClientUprobeStorageMap, m.HttpEvents, + m.HttpHeaders, m.SliceArrayBuffMap, m.TrackedSpans, m.TrackedSpansBySc, @@ -139,12 +143,14 @@ func (m *bpfMaps) Close() error { type bpfPrograms struct { UprobeTransportRoundTrip *ebpf.Program `ebpf:"uprobe_Transport_roundTrip"` UprobeTransportRoundTripReturns *ebpf.Program `ebpf:"uprobe_Transport_roundTrip_Returns"` + UprobeWriteSubset *ebpf.Program `ebpf:"uprobe_writeSubset"` } func (p *bpfPrograms) Close() error { return _BpfClose( p.UprobeTransportRoundTrip, p.UprobeTransportRoundTripReturns, + p.UprobeWriteSubset, ) } diff --git a/internal/pkg/instrumentation/bpf/net/http/client/bpf_no_tp_bpfel_arm64.go b/internal/pkg/instrumentation/bpf/net/http/client/bpf_no_tp_bpfel_arm64.go new file mode 100644 index 000000000..e22fd907e --- /dev/null +++ b/internal/pkg/instrumentation/bpf/net/http/client/bpf_no_tp_bpfel_arm64.go @@ -0,0 +1,169 @@ +// Code generated by bpf2go; DO NOT EDIT. +//go:build arm64 + +package client + +import ( + "bytes" + _ "embed" + "fmt" + "io" + + "github.com/cilium/ebpf" +) + +type bpf_no_tpHttpRequestT struct { + StartTime uint64 + EndTime uint64 + Sc bpf_no_tpSpanContext + Psc bpf_no_tpSpanContext + Host [256]int8 + Proto [8]int8 + StatusCode uint64 + Method [10]int8 + Path [100]int8 + _ [2]byte +} + +type bpf_no_tpSliceArrayBuff struct{ Buff [1024]uint8 } + +type bpf_no_tpSpanContext struct { + TraceID [16]uint8 + SpanID [8]uint8 +} + +// loadBpf_no_tp returns the embedded CollectionSpec for bpf_no_tp. +func loadBpf_no_tp() (*ebpf.CollectionSpec, error) { + reader := bytes.NewReader(_Bpf_no_tpBytes) + spec, err := ebpf.LoadCollectionSpecFromReader(reader) + if err != nil { + return nil, fmt.Errorf("can't load bpf_no_tp: %w", err) + } + + return spec, err +} + +// loadBpf_no_tpObjects loads bpf_no_tp and converts it into a struct. +// +// The following types are suitable as obj argument: +// +// *bpf_no_tpObjects +// *bpf_no_tpPrograms +// *bpf_no_tpMaps +// +// See ebpf.CollectionSpec.LoadAndAssign documentation for details. +func loadBpf_no_tpObjects(obj interface{}, opts *ebpf.CollectionOptions) error { + spec, err := loadBpf_no_tp() + if err != nil { + return err + } + + return spec.LoadAndAssign(obj, opts) +} + +// bpf_no_tpSpecs contains maps and programs before they are loaded into the kernel. +// +// It can be passed ebpf.CollectionSpec.Assign. +type bpf_no_tpSpecs struct { + bpf_no_tpProgramSpecs + bpf_no_tpMapSpecs +} + +// bpf_no_tpSpecs contains programs before they are loaded into the kernel. +// +// It can be passed ebpf.CollectionSpec.Assign. +type bpf_no_tpProgramSpecs struct { + UprobeTransportRoundTrip *ebpf.ProgramSpec `ebpf:"uprobe_Transport_roundTrip"` + UprobeTransportRoundTripReturns *ebpf.ProgramSpec `ebpf:"uprobe_Transport_roundTrip_Returns"` + UprobeWriteSubset *ebpf.ProgramSpec `ebpf:"uprobe_writeSubset"` +} + +// bpf_no_tpMapSpecs contains maps before they are loaded into the kernel. +// +// It can be passed ebpf.CollectionSpec.Assign. +type bpf_no_tpMapSpecs struct { + AllocMap *ebpf.MapSpec `ebpf:"alloc_map"` + Events *ebpf.MapSpec `ebpf:"events"` + GolangMapbucketStorageMap *ebpf.MapSpec `ebpf:"golang_mapbucket_storage_map"` + HttpClientUprobeStorageMap *ebpf.MapSpec `ebpf:"http_client_uprobe_storage_map"` + HttpEvents *ebpf.MapSpec `ebpf:"http_events"` + HttpHeaders *ebpf.MapSpec `ebpf:"http_headers"` + SliceArrayBuffMap *ebpf.MapSpec `ebpf:"slice_array_buff_map"` + TrackedSpans *ebpf.MapSpec `ebpf:"tracked_spans"` + TrackedSpansBySc *ebpf.MapSpec `ebpf:"tracked_spans_by_sc"` +} + +// bpf_no_tpObjects contains all objects after they have been loaded into the kernel. +// +// It can be passed to loadBpf_no_tpObjects or ebpf.CollectionSpec.LoadAndAssign. +type bpf_no_tpObjects struct { + bpf_no_tpPrograms + bpf_no_tpMaps +} + +func (o *bpf_no_tpObjects) Close() error { + return _Bpf_no_tpClose( + &o.bpf_no_tpPrograms, + &o.bpf_no_tpMaps, + ) +} + +// bpf_no_tpMaps contains all maps after they have been loaded into the kernel. +// +// It can be passed to loadBpf_no_tpObjects or ebpf.CollectionSpec.LoadAndAssign. +type bpf_no_tpMaps struct { + AllocMap *ebpf.Map `ebpf:"alloc_map"` + Events *ebpf.Map `ebpf:"events"` + GolangMapbucketStorageMap *ebpf.Map `ebpf:"golang_mapbucket_storage_map"` + HttpClientUprobeStorageMap *ebpf.Map `ebpf:"http_client_uprobe_storage_map"` + HttpEvents *ebpf.Map `ebpf:"http_events"` + HttpHeaders *ebpf.Map `ebpf:"http_headers"` + SliceArrayBuffMap *ebpf.Map `ebpf:"slice_array_buff_map"` + TrackedSpans *ebpf.Map `ebpf:"tracked_spans"` + TrackedSpansBySc *ebpf.Map `ebpf:"tracked_spans_by_sc"` +} + +func (m *bpf_no_tpMaps) Close() error { + return _Bpf_no_tpClose( + m.AllocMap, + m.Events, + m.GolangMapbucketStorageMap, + m.HttpClientUprobeStorageMap, + m.HttpEvents, + m.HttpHeaders, + m.SliceArrayBuffMap, + m.TrackedSpans, + m.TrackedSpansBySc, + ) +} + +// bpf_no_tpPrograms contains all programs after they have been loaded into the kernel. +// +// It can be passed to loadBpf_no_tpObjects or ebpf.CollectionSpec.LoadAndAssign. +type bpf_no_tpPrograms struct { + UprobeTransportRoundTrip *ebpf.Program `ebpf:"uprobe_Transport_roundTrip"` + UprobeTransportRoundTripReturns *ebpf.Program `ebpf:"uprobe_Transport_roundTrip_Returns"` + UprobeWriteSubset *ebpf.Program `ebpf:"uprobe_writeSubset"` +} + +func (p *bpf_no_tpPrograms) Close() error { + return _Bpf_no_tpClose( + p.UprobeTransportRoundTrip, + p.UprobeTransportRoundTripReturns, + p.UprobeWriteSubset, + ) +} + +func _Bpf_no_tpClose(closers ...io.Closer) error { + for _, closer := range closers { + if err := closer.Close(); err != nil { + return err + } + } + return nil +} + +// Do not access this directly. +// +//go:embed bpf_no_tp_bpfel_arm64.o +var _Bpf_no_tpBytes []byte diff --git a/internal/pkg/instrumentation/bpf/net/http/client/bpf_no_tp_bpfel_x86.go b/internal/pkg/instrumentation/bpf/net/http/client/bpf_no_tp_bpfel_x86.go new file mode 100644 index 000000000..1df381384 --- /dev/null +++ b/internal/pkg/instrumentation/bpf/net/http/client/bpf_no_tp_bpfel_x86.go @@ -0,0 +1,169 @@ +// Code generated by bpf2go; DO NOT EDIT. +//go:build 386 || amd64 + +package client + +import ( + "bytes" + _ "embed" + "fmt" + "io" + + "github.com/cilium/ebpf" +) + +type bpf_no_tpHttpRequestT struct { + StartTime uint64 + EndTime uint64 + Sc bpf_no_tpSpanContext + Psc bpf_no_tpSpanContext + Host [256]int8 + Proto [8]int8 + StatusCode uint64 + Method [10]int8 + Path [100]int8 + _ [2]byte +} + +type bpf_no_tpSliceArrayBuff struct{ Buff [1024]uint8 } + +type bpf_no_tpSpanContext struct { + TraceID [16]uint8 + SpanID [8]uint8 +} + +// loadBpf_no_tp returns the embedded CollectionSpec for bpf_no_tp. +func loadBpf_no_tp() (*ebpf.CollectionSpec, error) { + reader := bytes.NewReader(_Bpf_no_tpBytes) + spec, err := ebpf.LoadCollectionSpecFromReader(reader) + if err != nil { + return nil, fmt.Errorf("can't load bpf_no_tp: %w", err) + } + + return spec, err +} + +// loadBpf_no_tpObjects loads bpf_no_tp and converts it into a struct. +// +// The following types are suitable as obj argument: +// +// *bpf_no_tpObjects +// *bpf_no_tpPrograms +// *bpf_no_tpMaps +// +// See ebpf.CollectionSpec.LoadAndAssign documentation for details. +func loadBpf_no_tpObjects(obj interface{}, opts *ebpf.CollectionOptions) error { + spec, err := loadBpf_no_tp() + if err != nil { + return err + } + + return spec.LoadAndAssign(obj, opts) +} + +// bpf_no_tpSpecs contains maps and programs before they are loaded into the kernel. +// +// It can be passed ebpf.CollectionSpec.Assign. +type bpf_no_tpSpecs struct { + bpf_no_tpProgramSpecs + bpf_no_tpMapSpecs +} + +// bpf_no_tpSpecs contains programs before they are loaded into the kernel. +// +// It can be passed ebpf.CollectionSpec.Assign. +type bpf_no_tpProgramSpecs struct { + UprobeTransportRoundTrip *ebpf.ProgramSpec `ebpf:"uprobe_Transport_roundTrip"` + UprobeTransportRoundTripReturns *ebpf.ProgramSpec `ebpf:"uprobe_Transport_roundTrip_Returns"` + UprobeWriteSubset *ebpf.ProgramSpec `ebpf:"uprobe_writeSubset"` +} + +// bpf_no_tpMapSpecs contains maps before they are loaded into the kernel. +// +// It can be passed ebpf.CollectionSpec.Assign. +type bpf_no_tpMapSpecs struct { + AllocMap *ebpf.MapSpec `ebpf:"alloc_map"` + Events *ebpf.MapSpec `ebpf:"events"` + GolangMapbucketStorageMap *ebpf.MapSpec `ebpf:"golang_mapbucket_storage_map"` + HttpClientUprobeStorageMap *ebpf.MapSpec `ebpf:"http_client_uprobe_storage_map"` + HttpEvents *ebpf.MapSpec `ebpf:"http_events"` + HttpHeaders *ebpf.MapSpec `ebpf:"http_headers"` + SliceArrayBuffMap *ebpf.MapSpec `ebpf:"slice_array_buff_map"` + TrackedSpans *ebpf.MapSpec `ebpf:"tracked_spans"` + TrackedSpansBySc *ebpf.MapSpec `ebpf:"tracked_spans_by_sc"` +} + +// bpf_no_tpObjects contains all objects after they have been loaded into the kernel. +// +// It can be passed to loadBpf_no_tpObjects or ebpf.CollectionSpec.LoadAndAssign. +type bpf_no_tpObjects struct { + bpf_no_tpPrograms + bpf_no_tpMaps +} + +func (o *bpf_no_tpObjects) Close() error { + return _Bpf_no_tpClose( + &o.bpf_no_tpPrograms, + &o.bpf_no_tpMaps, + ) +} + +// bpf_no_tpMaps contains all maps after they have been loaded into the kernel. +// +// It can be passed to loadBpf_no_tpObjects or ebpf.CollectionSpec.LoadAndAssign. +type bpf_no_tpMaps struct { + AllocMap *ebpf.Map `ebpf:"alloc_map"` + Events *ebpf.Map `ebpf:"events"` + GolangMapbucketStorageMap *ebpf.Map `ebpf:"golang_mapbucket_storage_map"` + HttpClientUprobeStorageMap *ebpf.Map `ebpf:"http_client_uprobe_storage_map"` + HttpEvents *ebpf.Map `ebpf:"http_events"` + HttpHeaders *ebpf.Map `ebpf:"http_headers"` + SliceArrayBuffMap *ebpf.Map `ebpf:"slice_array_buff_map"` + TrackedSpans *ebpf.Map `ebpf:"tracked_spans"` + TrackedSpansBySc *ebpf.Map `ebpf:"tracked_spans_by_sc"` +} + +func (m *bpf_no_tpMaps) Close() error { + return _Bpf_no_tpClose( + m.AllocMap, + m.Events, + m.GolangMapbucketStorageMap, + m.HttpClientUprobeStorageMap, + m.HttpEvents, + m.HttpHeaders, + m.SliceArrayBuffMap, + m.TrackedSpans, + m.TrackedSpansBySc, + ) +} + +// bpf_no_tpPrograms contains all programs after they have been loaded into the kernel. +// +// It can be passed to loadBpf_no_tpObjects or ebpf.CollectionSpec.LoadAndAssign. +type bpf_no_tpPrograms struct { + UprobeTransportRoundTrip *ebpf.Program `ebpf:"uprobe_Transport_roundTrip"` + UprobeTransportRoundTripReturns *ebpf.Program `ebpf:"uprobe_Transport_roundTrip_Returns"` + UprobeWriteSubset *ebpf.Program `ebpf:"uprobe_writeSubset"` +} + +func (p *bpf_no_tpPrograms) Close() error { + return _Bpf_no_tpClose( + p.UprobeTransportRoundTrip, + p.UprobeTransportRoundTripReturns, + p.UprobeWriteSubset, + ) +} + +func _Bpf_no_tpClose(closers ...io.Closer) error { + for _, closer := range closers { + if err := closer.Close(); err != nil { + return err + } + } + return nil +} + +// Do not access this directly. +// +//go:embed bpf_no_tp_bpfel_x86.o +var _Bpf_no_tpBytes []byte diff --git a/internal/pkg/instrumentation/bpf/net/http/client/probe.go b/internal/pkg/instrumentation/bpf/net/http/client/probe.go index 8ccea8c19..9225a1681 100644 --- a/internal/pkg/instrumentation/bpf/net/http/client/probe.go +++ b/internal/pkg/instrumentation/bpf/net/http/client/probe.go @@ -38,6 +38,7 @@ import ( ) //go:generate go run github.com/cilium/ebpf/cmd/bpf2go -target amd64,arm64 -cc clang -cflags $CFLAGS bpf ./bpf/probe.bpf.c +//go:generate go run github.com/cilium/ebpf/cmd/bpf2go -target amd64,arm64 -cc clang -cflags $CFLAGS bpf_no_tp ./bpf/probe.bpf.c -- -DNO_HEADER_PROPAGATION const ( // pkg is the package being instrumented. @@ -92,12 +93,24 @@ func New(logger logr.Logger) probe.Probe { Key: "request_proto_pos", Val: structfield.NewID("std", "net/http", "Request", "Proto"), }, + probe.StructFieldConst{ + Key: "io_writer_buf_ptr_pos", + Val: structfield.NewID("std", "bufio", "Writer", "buf"), + }, + probe.StructFieldConst{ + Key: "io_writer_n_pos", + Val: structfield.NewID("std", "bufio", "Writer", "n"), + }, }, Uprobes: []probe.Uprobe[bpfObjects]{ { Sym: "net/http.(*Transport).roundTrip", Fn: uprobeRoundTrip, }, + { + Sym: "net/http.Header.writeSubset", + Fn: uprobeWriteSubset, + }, }, ReaderFn: func(obj bpfObjects) (*perf.Reader, error) { @@ -110,7 +123,8 @@ func New(logger logr.Logger) probe.Probe { func verifyAndLoadBpf() (*ebpf.CollectionSpec, error) { if !utils.SupportsContextPropagation() { - return nil, fmt.Errorf("the Linux Kernel doesn't support context propagation, please check if the kernel is in lockdown mode (/sys/kernel/security/lockdown)") + fmt.Fprintf(os.Stderr, "the Linux Kernel doesn't support context propagation, please check if the kernel is in lockdown mode (/sys/kernel/security/lockdown)") + return loadBpf_no_tp() } return loadBpf() @@ -146,6 +160,21 @@ func uprobeRoundTrip(name string, exec *link.Executable, target *process.TargetD return links, nil } +func uprobeWriteSubset(name string, exec *link.Executable, target *process.TargetDetails, obj *bpfObjects) ([]link.Link, error) { + offset, err := target.GetFunctionOffset(name) + if err != nil { + return nil, err + } + + opts := &link.UprobeOptions{Address: offset} + l, err := exec.Uprobe("", obj.UprobeWriteSubset, opts) + if err != nil { + return nil, err + } + + return []link.Link{l}, nil +} + // event represents an event in an HTTP server during an HTTP // request-response. type event struct { diff --git a/internal/tools/inspect/cmd/offsetgen/main.go b/internal/tools/inspect/cmd/offsetgen/main.go index c79daf481..5d3e092ae 100644 --- a/internal/tools/inspect/cmd/offsetgen/main.go +++ b/internal/tools/inspect/cmd/offsetgen/main.go @@ -118,6 +118,8 @@ func manifests() ([]inspect.Manifest, error) { structfield.NewID("std", "net/http", "Request", "RequestURI"), structfield.NewID("std", "net/http", "Request", "Host"), structfield.NewID("std", "net/url", "URL", "Path"), + structfield.NewID("std", "bufio", "Writer", "buf"), + structfield.NewID("std", "bufio", "Writer", "n"), }, }, { diff --git a/internal/tools/inspect/cmd/offsetgen/verions.go b/internal/tools/inspect/cmd/offsetgen/versions.go similarity index 100% rename from internal/tools/inspect/cmd/offsetgen/verions.go rename to internal/tools/inspect/cmd/offsetgen/versions.go From f612e89dad6a0f3774a246d8a2a3ebc3d7f03dc2 Mon Sep 17 00:00:00 2001 From: Nikola Grcevski Date: Mon, 4 Mar 2024 16:10:16 -0500 Subject: [PATCH 02/14] Remove old code, optional probes --- .../bpf/net/http/client/bpf/probe.bpf.c | 109 ------------------ .../bpf/net/http/client/probe.go | 5 +- internal/pkg/instrumentation/manager.go | 8 +- .../pkg/instrumentation/probe/manifest.go | 14 ++- .../instrumentation/probe/manifest_test.go | 8 +- internal/pkg/instrumentation/probe/probe.go | 4 +- 6 files changed, 27 insertions(+), 121 deletions(-) diff --git a/internal/pkg/instrumentation/bpf/net/http/client/bpf/probe.bpf.c b/internal/pkg/instrumentation/bpf/net/http/client/bpf/probe.bpf.c index 54b88c81a..a29907333 100644 --- a/internal/pkg/instrumentation/bpf/net/http/client/bpf/probe.bpf.c +++ b/internal/pkg/instrumentation/bpf/net/http/client/bpf/probe.bpf.c @@ -69,109 +69,6 @@ volatile const u64 request_proto_pos; volatile const u64 io_writer_buf_ptr_pos; volatile const u64 io_writer_n_pos; -#ifndef NO_HEADER_PROPAGATION -static __always_inline long inject_header(void* headers_ptr, struct span_context* propagated_ctx) { - // Read the key-value count - this field must be the first one in the hmap struct as documented in src/runtime/map.go - u64 curr_keyvalue_count = 0; - long res = bpf_probe_read_user(&curr_keyvalue_count, sizeof(curr_keyvalue_count), headers_ptr); - - if (res < 0) { - bpf_printk("Couldn't read map key-value count from user"); - return -1; - } - - if (curr_keyvalue_count >= 8) { - bpf_printk("Map size is bigger than 8, skipping context propagation"); - return -1; - } - - // Get pointer to temp bucket struct we store in a map (avoiding large local variable on the stack) - // Performing read-modify-write on the bucket - u32 map_id = 0; - struct map_bucket *bucket_map_value = bpf_map_lookup_elem(&golang_mapbucket_storage_map, &map_id); - if (!bucket_map_value) { - return -1; - } - - void *buckets_ptr_ptr = (void*) (headers_ptr + buckets_ptr_pos); - void *bucket_ptr = 0; // The actual pointer to the buckets - - if (curr_keyvalue_count == 0) { - // No key-value pairs in the Go map, need to "allocate" memory for the user - bucket_ptr = write_target_data(bucket_map_value, sizeof(struct map_bucket)); - if (bucket_ptr == NULL) { - bpf_printk("inject_header: Failed to write bucket to user"); - return -1; - } - // Update the buckets pointer in the hmap struct to point to newly allocated bucket - res = bpf_probe_write_user(buckets_ptr_ptr, &bucket_ptr, sizeof(bucket_ptr)); - if (res < 0) { - bpf_printk("Failed to update the map bucket pointer for the user"); - return -1; - } - } else { - // There is at least 1 key-value pair, hence the bucket pointer from the user is valid - // Read the bucket pointer - res = bpf_probe_read_user(&bucket_ptr, sizeof(bucket_ptr), buckets_ptr_ptr); - // Read the user's bucket to the eBPF map entry - bpf_probe_read_user(bucket_map_value, sizeof(struct map_bucket), bucket_ptr); - } - - u8 bucket_index = curr_keyvalue_count & 0x7; - - char traceparent_tophash = 0xee; - bucket_map_value->tophash[bucket_index] = traceparent_tophash; - - // Prepare the key string for the user - char key[W3C_KEY_LENGTH] = "traceparent"; - void *ptr = write_target_data(key, W3C_KEY_LENGTH); - if (ptr == NULL) { - bpf_printk("inject_header: Failed to write key to user"); - return -1; - } - bucket_map_value->keys[bucket_index] = (struct go_string) {.len = W3C_KEY_LENGTH, .str = ptr}; - - // Prepare the value string slice - // First the value string which constains the span context - char val[W3C_VAL_LENGTH]; - span_context_to_w3c_string(propagated_ctx, val); - ptr = write_target_data(val, sizeof(val)); - if(ptr == NULL) { - bpf_printk("inject_header: Failed to write value to user"); - return -1; - } - - // The go string pointing to the above val - struct go_string header_value = {.len = W3C_VAL_LENGTH, .str = ptr}; - ptr = write_target_data((void*)&header_value, sizeof(header_value)); - if(ptr == NULL) { - bpf_printk("inject_header: Failed to write go_string to user"); - return -1; - } - - // Last, go_slice pointing to the above go_string - bucket_map_value->values[bucket_index] = (struct go_slice) {.array = ptr, .cap = 1, .len = 1}; - - // Update the map header count field - curr_keyvalue_count += 1; - res = bpf_probe_write_user(headers_ptr, &curr_keyvalue_count, sizeof(curr_keyvalue_count)); - if (res < 0) { - bpf_printk("Failed to update key-value count in map header"); - return -1; - } - - // Update the bucket - res = bpf_probe_write_user(bucket_ptr, bucket_map_value, sizeof(struct map_bucket)); - if (res < 0) { - bpf_printk("Failed to update bucket content"); - return -1; - } - - bpf_memset((unsigned char *)bucket_map_value, sizeof(struct map_bucket), 0); - return 0; -} -#endif - // This instrumentation attaches uprobe to the following function: // func net/http/transport.roundTrip(req *Request) (*Response, error) SEC("uprobe/Transport_roundTrip") @@ -244,12 +141,6 @@ int uprobe_Transport_roundTrip(struct pt_regs *ctx) { bpf_map_update_elem(&http_headers, &headers_ptr, &key, 0); } -#ifndef NO_HEADER_PROPAGATION - long res = inject_header(headers_ptr, &httpReq->sc); - if (res < 0) { - bpf_printk("uprobe_Transport_roundTrip: Failed to inject header"); - } -#endif // Write event bpf_map_update_elem(&http_events, &key, httpReq, 0); start_tracking_span(context_ptr_val, &httpReq->sc); diff --git a/internal/pkg/instrumentation/bpf/net/http/client/probe.go b/internal/pkg/instrumentation/bpf/net/http/client/probe.go index 9225a1681..987b7ab27 100644 --- a/internal/pkg/instrumentation/bpf/net/http/client/probe.go +++ b/internal/pkg/instrumentation/bpf/net/http/client/probe.go @@ -108,8 +108,9 @@ func New(logger logr.Logger) probe.Probe { Fn: uprobeRoundTrip, }, { - Sym: "net/http.Header.writeSubset", - Fn: uprobeWriteSubset, + Sym: "net/http.Header.writeSubset", + Fn: uprobeWriteSubset, + Optional: true, }, }, diff --git a/internal/pkg/instrumentation/manager.go b/internal/pkg/instrumentation/manager.go index 446c744e5..a69515c2b 100644 --- a/internal/pkg/instrumentation/manager.go +++ b/internal/pkg/instrumentation/manager.go @@ -85,7 +85,7 @@ func (m *Manager) GetRelevantFuncs() map[string]interface{} { funcsMap := make(map[string]interface{}) for _, i := range m.probes { for _, s := range i.Manifest().Symbols { - funcsMap[s] = nil + funcsMap[s.Symbol] = nil } } @@ -103,8 +103,10 @@ func (m *Manager) FilterUnusedProbes(target *process.TargetDetails) { for name, inst := range m.probes { funcsFound := 0 for _, s := range inst.Manifest().Symbols { - if _, exists := existingFuncMap[s]; exists { - funcsFound++ + if !s.Optional { + if _, exists := existingFuncMap[s.Symbol]; exists { + funcsFound++ + } } } diff --git a/internal/pkg/instrumentation/probe/manifest.go b/internal/pkg/instrumentation/probe/manifest.go index 2d6e11ef5..bb6de7789 100644 --- a/internal/pkg/instrumentation/probe/manifest.go +++ b/internal/pkg/instrumentation/probe/manifest.go @@ -23,6 +23,12 @@ import ( "go.opentelemetry.io/auto/internal/pkg/structfield" ) +// Wrapper object for the Manifest function symbol +type FunctionSymbol struct { + Symbol string + Optional bool +} + // Manifest contains information about a package being instrumented. type Manifest struct { // ID is a unique identifier for the probe. @@ -34,7 +40,7 @@ type Manifest struct { // Symbols are the runtime symbols that are used to attach a probe's eBPF // program to a perf events. - Symbols []string + Symbols []FunctionSymbol } // ID is a unique identifier for a probe. @@ -52,7 +58,7 @@ func (id ID) String() string { // NewManifest returns a new Manifest for the instrumentation probe with name // that instruments pkg. The structfields and symbols will be sorted in-place // and added directly to the returned Manifest. -func NewManifest(id ID, structfields []structfield.ID, symbols []string) Manifest { +func NewManifest(id ID, structfields []structfield.ID, symbols []FunctionSymbol) Manifest { sort.Slice(structfields, func(i, j int) bool { if structfields[i].ModPath == structfields[j].ModPath { if structfields[i].PkgPath == structfields[j].PkgPath { @@ -66,7 +72,9 @@ func NewManifest(id ID, structfields []structfield.ID, symbols []string) Manifes return structfields[i].ModPath < structfields[j].ModPath }) - sort.Strings(symbols) + sort.Slice(symbols, func(i, j int) bool { + return symbols[i].Symbol < symbols[j].Symbol + }) return Manifest{ Id: id, diff --git a/internal/pkg/instrumentation/probe/manifest_test.go b/internal/pkg/instrumentation/probe/manifest_test.go index b6ce2c1b7..912d6b093 100644 --- a/internal/pkg/instrumentation/probe/manifest_test.go +++ b/internal/pkg/instrumentation/probe/manifest_test.go @@ -23,6 +23,10 @@ import ( "go.opentelemetry.io/auto/internal/pkg/structfield" ) +func fs(s string) FunctionSymbol { + return FunctionSymbol{Symbol: s, Optional: false} +} + func TestNewManifest(t *testing.T) { const ( spanKind = trace.SpanKindServer @@ -48,12 +52,12 @@ func TestNewManifest(t *testing.T) { got := NewManifest( ID{spanKind, pkg}, []structfield.ID{sAABB, sABAA, sAAAA, sAAAC, sBAAA, sAAAB, sAABA, sAABC}, - []string{d, a, c, b}, + []FunctionSymbol{fs(d), fs(a), fs(c), fs(b)}, ) want := Manifest{ Id: ID{spanKind, pkg}, StructFields: []structfield.ID{sAAAA, sAAAB, sAAAC, sAABA, sAABB, sAABC, sABAA, sBAAA}, - Symbols: []string{a, b, c, d}, + Symbols: []FunctionSymbol{fs(a), fs(b), fs(c), fs(d)}, } assert.Equal(t, want, got) } diff --git a/internal/pkg/instrumentation/probe/probe.go b/internal/pkg/instrumentation/probe/probe.go index 5e77599ed..1224d93fe 100644 --- a/internal/pkg/instrumentation/probe/probe.go +++ b/internal/pkg/instrumentation/probe/probe.go @@ -86,9 +86,9 @@ type Base[BPFObj any, BPFEvent any] struct { func (i *Base[BPFObj, BPFEvent]) Manifest() Manifest { structfields := consts(i.Consts).structFields() - symbols := make([]string, 0, len(i.Uprobes)) + symbols := make([]FunctionSymbol, 0, len(i.Uprobes)) for _, up := range i.Uprobes { - symbols = append(symbols, up.Sym) + symbols = append(symbols, FunctionSymbol{Symbol: up.Sym, Optional: up.Optional}) } return NewManifest(i.ID, structfields, symbols) From 63e1dce75f3dd12a43365fa3298741d122202a9e Mon Sep 17 00:00:00 2001 From: Nikola Grcevski Date: Mon, 1 Apr 2024 16:27:53 -0400 Subject: [PATCH 03/14] update offsets --- internal/pkg/inject/offset_results.json | 169 ++++++++++++------------ 1 file changed, 84 insertions(+), 85 deletions(-) diff --git a/internal/pkg/inject/offset_results.json b/internal/pkg/inject/offset_results.json index 70dbe74e7..9a4b25006 100644 --- a/internal/pkg/inject/offset_results.json +++ b/internal/pkg/inject/offset_results.json @@ -1490,16 +1490,16 @@ "module": "std", "packages": [ { - "package": "net/http", + "package": "bufio", "structs": [ { - "struct": "Request", + "struct": "Writer", "fields": [ { - "field": "Header", + "field": "buf", "offsets": [ { - "offset": 56, + "offset": 16, "versions": [ "1.12.0", "1.12.1", @@ -1653,10 +1653,10 @@ ] }, { - "field": "Host", + "field": "n", "offsets": [ { - "offset": 128, + "offset": 40, "versions": [ "1.12.0", "1.12.1", @@ -1808,12 +1808,22 @@ ] } ] - }, + } + ] + } + ] + }, + { + "package": "net/http", + "structs": [ + { + "struct": "Request", + "fields": [ { - "field": "Method", + "field": "Header", "offsets": [ { - "offset": 0, + "offset": 56, "versions": [ "1.12.0", "1.12.1", @@ -1967,10 +1977,10 @@ ] }, { - "field": "Proto", + "field": "Host", "offsets": [ { - "offset": 24, + "offset": 128, "versions": [ "1.12.0", "1.12.1", @@ -2124,10 +2134,10 @@ ] }, { - "field": "RemoteAddr", + "field": "Method", "offsets": [ { - "offset": 176, + "offset": 0, "versions": [ "1.12.0", "1.12.1", @@ -2281,10 +2291,10 @@ ] }, { - "field": "RequestURI", + "field": "Proto", "offsets": [ { - "offset": 192, + "offset": 24, "versions": [ "1.12.0", "1.12.1", @@ -2438,10 +2448,10 @@ ] }, { - "field": "URL", + "field": "RemoteAddr", "offsets": [ { - "offset": 16, + "offset": 176, "versions": [ "1.12.0", "1.12.1", @@ -2595,10 +2605,10 @@ ] }, { - "field": "ctx", + "field": "RequestURI", "offsets": [ { - "offset": 232, + "offset": 192, "versions": [ "1.12.0", "1.12.1", @@ -2750,14 +2760,9 @@ ] } ] - } - ] - }, - { - "struct": "Response", - "fields": [ + }, { - "field": "StatusCode", + "field": "URL", "offsets": [ { "offset": 16, @@ -2912,17 +2917,12 @@ ] } ] - } - ] - }, - { - "struct": "response", - "fields": [ + }, { - "field": "req", + "field": "ctx", "offsets": [ { - "offset": 8, + "offset": 232, "versions": [ "1.12.0", "1.12.1", @@ -3074,12 +3074,17 @@ ] } ] - }, + } + ] + }, + { + "struct": "Response", + "fields": [ { - "field": "status", + "field": "StatusCode", "offsets": [ { - "offset": 112, + "offset": 16, "versions": [ "1.12.0", "1.12.1", @@ -3112,19 +3117,14 @@ "1.13.10", "1.13.11", "1.13.12", + "1.13.13", + "1.13.14", + "1.13.15", "1.14.0", "1.14.1", "1.14.2", "1.14.3", - "1.14.4" - ] - }, - { - "offset": 120, - "versions": [ - "1.13.13", - "1.13.14", - "1.13.15", + "1.14.4", "1.14.5", "1.14.6", "1.14.7", @@ -3238,20 +3238,15 @@ ] } ] - } - ] - }, - { - "package": "net/url", - "structs": [ + }, { - "struct": "URL", + "struct": "response", "fields": [ { - "field": "Path", + "field": "req", "offsets": [ { - "offset": 56, + "offset": 8, "versions": [ "1.12.0", "1.12.1", @@ -3403,22 +3398,12 @@ ] } ] - } - ] - } - ] - }, - { - "package": "runtime", - "structs": [ - { - "struct": "g", - "fields": [ + }, { - "field": "goid", + "field": "status", "offsets": [ { - "offset": 152, + "offset": 112, "versions": [ "1.12.0", "1.12.1", @@ -3451,14 +3436,19 @@ "1.13.10", "1.13.11", "1.13.12", - "1.13.13", - "1.13.14", - "1.13.15", "1.14.0", "1.14.1", "1.14.2", "1.14.3", - "1.14.4", + "1.14.4" + ] + }, + { + "offset": 120, + "versions": [ + "1.13.13", + "1.13.14", + "1.13.15", "1.14.5", "1.14.6", "1.14.7", @@ -3572,15 +3562,20 @@ ] } ] - }, + } + ] + }, + { + "package": "net/url", + "structs": [ { - "struct": "hmap", + "struct": "URL", "fields": [ { - "field": "buckets", + "field": "Path", "offsets": [ { - "offset": 16, + "offset": 56, "versions": [ "1.12.0", "1.12.1", @@ -3738,16 +3733,16 @@ ] }, { - "package": "bufio", + "package": "runtime", "structs": [ { - "struct": "Writer", + "struct": "g", "fields": [ { - "field": "buf", + "field": "goid", "offsets": [ { - "offset": 16, + "offset": 152, "versions": [ "1.12.0", "1.12.1", @@ -3893,7 +3888,9 @@ "1.21.5", "1.21.6", "1.21.7", - "1.22.0" + "1.21.8", + "1.22.0", + "1.22.1" ] } ] @@ -3901,13 +3898,13 @@ ] }, { - "struct": "Writer", + "struct": "hmap", "fields": [ { - "field": "n", + "field": "buckets", "offsets": [ { - "offset": 40, + "offset": 16, "versions": [ "1.12.0", "1.12.1", @@ -4053,7 +4050,9 @@ "1.21.5", "1.21.6", "1.21.7", - "1.22.0" + "1.21.8", + "1.22.0", + "1.22.1" ] } ] From b0a1b8a74d230440b8b023ca428c77282e77d01a Mon Sep 17 00:00:00 2001 From: Nikola Grcevski Date: Mon, 1 Apr 2024 20:10:27 -0400 Subject: [PATCH 04/14] remove prints, add dependents probe filtering --- .../bpf/net/http/client/bpf/probe.bpf.c | 8 -------- .../instrumentation/bpf/net/http/client/probe.go | 5 +++-- internal/pkg/instrumentation/manager.go | 15 ++++++++++++--- internal/pkg/instrumentation/probe/manifest.go | 14 +++++++++++--- .../pkg/instrumentation/probe/manifest_test.go | 8 ++++++-- internal/pkg/instrumentation/probe/probe.go | 7 ++++--- 6 files changed, 36 insertions(+), 21 deletions(-) diff --git a/internal/pkg/instrumentation/bpf/net/http/client/bpf/probe.bpf.c b/internal/pkg/instrumentation/bpf/net/http/client/bpf/probe.bpf.c index 54b88c81a..1069eae3f 100644 --- a/internal/pkg/instrumentation/bpf/net/http/client/bpf/probe.bpf.c +++ b/internal/pkg/instrumentation/bpf/net/http/client/bpf/probe.bpf.c @@ -240,7 +240,6 @@ int uprobe_Transport_roundTrip(struct pt_regs *ctx) { void *headers_ptr = 0; bpf_probe_read(&headers_ptr, sizeof(headers_ptr), (void *)(req_ptr+headers_ptr_pos)); if (headers_ptr) { - bpf_printk("headers_ptr = %llx, key = %llx", headers_ptr, key); bpf_map_update_elem(&http_headers, &headers_ptr, &key, 0); } @@ -294,20 +293,15 @@ int uprobe_writeSubset(struct pt_regs *ctx) { u64 io_writer_pos = 3; void *io_writer_ptr = get_argument(ctx, io_writer_pos); - bpf_printk("headers_ptr = %llx, io_writer_ptr = %llx", headers_ptr, io_writer_ptr); - void **key_ptr = bpf_map_lookup_elem(&http_headers, &headers_ptr); if (key_ptr) { void *key = *key_ptr; - bpf_printk("Found request key %llx", key); struct http_request_t *http_req_span = bpf_map_lookup_elem(&http_events, &key); if (http_req_span) { char tp[W3C_VAL_LENGTH]; span_context_to_w3c_string(&http_req_span->sc, tp); - bpf_printk("Will write %s", tp); - void *buf_ptr = 0; bpf_probe_read(&buf_ptr, sizeof(buf_ptr), (void *)(io_writer_ptr + io_writer_buf_ptr_pos)); // grab buf ptr if (!buf_ptr) { @@ -320,8 +314,6 @@ int uprobe_writeSubset(struct pt_regs *ctx) { s64 len = 0; bpf_probe_read(&len, sizeof(s64), (void *)(io_writer_ptr + io_writer_n_pos)); // grab len - bpf_printk("buf_ptr %llx, len=%d, size=%d", (void*)buf_ptr, len, size); - #ifndef NO_HEADER_PROPAGATION if (len < (size - W3C_VAL_LENGTH - W3C_KEY_LENGTH - 4)) { // 4 = strlen(":_") + strlen("\r\n") char key[W3C_KEY_LENGTH + 2] = "Traceparent: "; diff --git a/internal/pkg/instrumentation/bpf/net/http/client/probe.go b/internal/pkg/instrumentation/bpf/net/http/client/probe.go index c3aa588ad..61ce6c3d0 100644 --- a/internal/pkg/instrumentation/bpf/net/http/client/probe.go +++ b/internal/pkg/instrumentation/bpf/net/http/client/probe.go @@ -107,8 +107,9 @@ func New(logger logr.Logger) probe.Probe { Fn: uprobeRoundTrip, }, { - Sym: "net/http.Header.writeSubset", - Fn: uprobeWriteSubset, + Sym: "net/http.Header.writeSubset", + Fn: uprobeWriteSubset, + DependsOn: []string{"net/http.(*Transport).roundTrip"}, }, }, diff --git a/internal/pkg/instrumentation/manager.go b/internal/pkg/instrumentation/manager.go index 446c744e5..66388f78b 100644 --- a/internal/pkg/instrumentation/manager.go +++ b/internal/pkg/instrumentation/manager.go @@ -85,7 +85,7 @@ func (m *Manager) GetRelevantFuncs() map[string]interface{} { funcsMap := make(map[string]interface{}) for _, i := range m.probes { for _, s := range i.Manifest().Symbols { - funcsMap[s] = nil + funcsMap[s.Symbol] = nil } } @@ -103,8 +103,17 @@ func (m *Manager) FilterUnusedProbes(target *process.TargetDetails) { for name, inst := range m.probes { funcsFound := 0 for _, s := range inst.Manifest().Symbols { - if _, exists := existingFuncMap[s]; exists { - funcsFound++ + if len(s.DependsOn) != 0 { + for _, depends := range s.DependsOn { + if _, exists := existingFuncMap[depends]; exists { + funcsFound++ + break + } + } + } else { + if _, exists := existingFuncMap[s.Symbol]; exists { + funcsFound++ + } } } diff --git a/internal/pkg/instrumentation/probe/manifest.go b/internal/pkg/instrumentation/probe/manifest.go index 2d6e11ef5..0602690a7 100644 --- a/internal/pkg/instrumentation/probe/manifest.go +++ b/internal/pkg/instrumentation/probe/manifest.go @@ -23,6 +23,12 @@ import ( "go.opentelemetry.io/auto/internal/pkg/structfield" ) +// Wrapper object for the Manifest function symbol. +type FunctionSymbol struct { + Symbol string + DependsOn []string +} + // Manifest contains information about a package being instrumented. type Manifest struct { // ID is a unique identifier for the probe. @@ -34,7 +40,7 @@ type Manifest struct { // Symbols are the runtime symbols that are used to attach a probe's eBPF // program to a perf events. - Symbols []string + Symbols []FunctionSymbol } // ID is a unique identifier for a probe. @@ -52,7 +58,7 @@ func (id ID) String() string { // NewManifest returns a new Manifest for the instrumentation probe with name // that instruments pkg. The structfields and symbols will be sorted in-place // and added directly to the returned Manifest. -func NewManifest(id ID, structfields []structfield.ID, symbols []string) Manifest { +func NewManifest(id ID, structfields []structfield.ID, symbols []FunctionSymbol) Manifest { sort.Slice(structfields, func(i, j int) bool { if structfields[i].ModPath == structfields[j].ModPath { if structfields[i].PkgPath == structfields[j].PkgPath { @@ -66,7 +72,9 @@ func NewManifest(id ID, structfields []structfield.ID, symbols []string) Manifes return structfields[i].ModPath < structfields[j].ModPath }) - sort.Strings(symbols) + sort.Slice(symbols, func(i, j int) bool { + return symbols[i].Symbol < symbols[j].Symbol + }) return Manifest{ Id: id, diff --git a/internal/pkg/instrumentation/probe/manifest_test.go b/internal/pkg/instrumentation/probe/manifest_test.go index b6ce2c1b7..b90501d50 100644 --- a/internal/pkg/instrumentation/probe/manifest_test.go +++ b/internal/pkg/instrumentation/probe/manifest_test.go @@ -23,6 +23,10 @@ import ( "go.opentelemetry.io/auto/internal/pkg/structfield" ) +func fs(s string) FunctionSymbol { + return FunctionSymbol{Symbol: s, DependsOn: nil} +} + func TestNewManifest(t *testing.T) { const ( spanKind = trace.SpanKindServer @@ -48,12 +52,12 @@ func TestNewManifest(t *testing.T) { got := NewManifest( ID{spanKind, pkg}, []structfield.ID{sAABB, sABAA, sAAAA, sAAAC, sBAAA, sAAAB, sAABA, sAABC}, - []string{d, a, c, b}, + []FunctionSymbol{fs(d), fs(a), fs(c), fs(b)}, ) want := Manifest{ Id: ID{spanKind, pkg}, StructFields: []structfield.ID{sAAAA, sAAAB, sAAAC, sAABA, sAABB, sAABC, sABAA, sBAAA}, - Symbols: []string{a, b, c, d}, + Symbols: []FunctionSymbol{fs(a), fs(b), fs(c), fs(d)}, } assert.Equal(t, want, got) } diff --git a/internal/pkg/instrumentation/probe/probe.go b/internal/pkg/instrumentation/probe/probe.go index 5e77599ed..b12db8360 100644 --- a/internal/pkg/instrumentation/probe/probe.go +++ b/internal/pkg/instrumentation/probe/probe.go @@ -86,9 +86,9 @@ type Base[BPFObj any, BPFEvent any] struct { func (i *Base[BPFObj, BPFEvent]) Manifest() Manifest { structfields := consts(i.Consts).structFields() - symbols := make([]string, 0, len(i.Uprobes)) + symbols := make([]FunctionSymbol, 0, len(i.Uprobes)) for _, up := range i.Uprobes { - symbols = append(symbols, up.Sym) + symbols = append(symbols, FunctionSymbol{Symbol: up.Sym, DependsOn: up.DependsOn}) } return NewManifest(i.ID, structfields, symbols) @@ -235,7 +235,8 @@ type Uprobe[BPFObj any] struct { // Optional is a boolean flag informing if the Uprobe is optional. If the // Uprobe is optional and fails to attach, the error is logged and // processing continues. - Optional bool + Optional bool + DependsOn []string } // Const is an constant that needs to be injected into an eBPF program. From 6675f9a83deafc69ec1374bb632964929ba3cc5d Mon Sep 17 00:00:00 2001 From: Nikola Grcevski Date: Mon, 1 Apr 2024 20:18:32 -0400 Subject: [PATCH 05/14] Add changelog statement --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 796a5c504..949a030f0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ OpenTelemetry Go Automatic Instrumentation adheres to [Semantic Versioning](http - Base Dockerfile and build caching ([#683](https://github.com/open-telemetry/opentelemetry-go-instrumentation/pull/683)) - Add `server.address`, `server.port` and `network.protocol.version` to HTTP client spans ([#696](https://github.com/open-telemetry/opentelemetry-go-instrumentation/pull/696)) - Update http server attributes to latest semantic conventions ([#708](https://github.com/open-telemetry/opentelemetry-go-instrumentation/pull/708)) +- Don't use external memory for http client header injection ([#705](https://github.com/open-telemetry/opentelemetry-go-instrumentation/pull/705)) ### Fixed From 6ea68b432a98e774f605cb766182de8be5623a8a Mon Sep 17 00:00:00 2001 From: Nikola Grcevski Date: Mon, 1 Apr 2024 20:41:09 -0400 Subject: [PATCH 06/14] add manager tests --- internal/pkg/instrumentation/manager_test.go | 100 +++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100644 internal/pkg/instrumentation/manager_test.go diff --git a/internal/pkg/instrumentation/manager_test.go b/internal/pkg/instrumentation/manager_test.go new file mode 100644 index 000000000..f57418dc2 --- /dev/null +++ b/internal/pkg/instrumentation/manager_test.go @@ -0,0 +1,100 @@ +package instrumentation + +import ( + "log" + "os" + "testing" + + "github.com/go-logr/stdr" + "github.com/hashicorp/go-version" + "github.com/stretchr/testify/assert" + "go.opentelemetry.io/auto/internal/pkg/process" + "go.opentelemetry.io/auto/internal/pkg/process/binary" +) + +func TestProbeFiltering(t *testing.T) { + ver, err := version.NewVersion("1.20.0") + assert.NoError(t, err) + + t.Run("empty target details", func(t *testing.T) { + m := fakeManager(t) + + td := process.TargetDetails{ + PID: 1, + Functions: []*binary.Func{}, + GoVersion: ver, + Libraries: map[string]*version.Version{}, + AllocationDetails: nil, + } + m.FilterUnusedProbes(&td) + assert.Equal(t, 0, len(m.probes)) + }) + + t.Run("only HTTP client target details", func(t *testing.T) { + m := fakeManager(t) + + httpFuncs := []*binary.Func{ + {Name: "net/http.(*Transport).roundTrip"}, + } + + td := process.TargetDetails{ + PID: 1, + Functions: httpFuncs, + GoVersion: ver, + Libraries: map[string]*version.Version{}, + AllocationDetails: nil, + } + m.FilterUnusedProbes(&td) + assert.Equal(t, 1, len(m.probes)) // one function, single probe + }) + + t.Run("HTTP server and client target details", func(t *testing.T) { + m := fakeManager(t) + + httpFuncs := []*binary.Func{ + {Name: "net/http.(*Transport).roundTrip"}, + {Name: "net/http.serverHandler.ServeHTTP"}, + } + + td := process.TargetDetails{ + PID: 1, + Functions: httpFuncs, + GoVersion: ver, + Libraries: map[string]*version.Version{}, + AllocationDetails: nil, + } + m.FilterUnusedProbes(&td) + assert.Equal(t, 2, len(m.probes)) + }) + + t.Run("HTTP server and client dependent function only target details", func(t *testing.T) { + m := fakeManager(t) + + httpFuncs := []*binary.Func{ + // writeSubset depends on "net/http.(*Transport).roundTrip", it should be ignored without roundTrip + {Name: "net/http.Header.writeSubset"}, + {Name: "net/http.serverHandler.ServeHTTP"}, + } + + td := process.TargetDetails{ + PID: 1, + Functions: httpFuncs, + GoVersion: ver, + Libraries: map[string]*version.Version{}, + AllocationDetails: nil, + } + m.FilterUnusedProbes(&td) + assert.Equal(t, 1, len(m.probes)) + }) +} + +func fakeManager(t *testing.T) *Manager { + logger := stdr.New(log.New(os.Stderr, "", log.LstdFlags)) + logger = logger.WithName("Instrumentation") + + m, err := NewManager(logger, nil, true) + assert.NoError(t, err) + assert.NotNil(t, m) + + return m +} From 5405fd43e8fb6f2305936e07e7ac63ad31275c67 Mon Sep 17 00:00:00 2001 From: Nikola Grcevski Date: Mon, 1 Apr 2024 20:45:00 -0400 Subject: [PATCH 07/14] add license to manager tests --- internal/pkg/instrumentation/manager_test.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/internal/pkg/instrumentation/manager_test.go b/internal/pkg/instrumentation/manager_test.go index f57418dc2..91b9921ee 100644 --- a/internal/pkg/instrumentation/manager_test.go +++ b/internal/pkg/instrumentation/manager_test.go @@ -1,3 +1,17 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package instrumentation import ( From 035ade8d88ae1db4551e95025a44a1f3670c8e84 Mon Sep 17 00:00:00 2001 From: Nikola Grcevski Date: Mon, 1 Apr 2024 20:51:48 -0400 Subject: [PATCH 08/14] format new test file --- internal/pkg/instrumentation/manager_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/pkg/instrumentation/manager_test.go b/internal/pkg/instrumentation/manager_test.go index 91b9921ee..55973e4d8 100644 --- a/internal/pkg/instrumentation/manager_test.go +++ b/internal/pkg/instrumentation/manager_test.go @@ -22,6 +22,7 @@ import ( "github.com/go-logr/stdr" "github.com/hashicorp/go-version" "github.com/stretchr/testify/assert" + "go.opentelemetry.io/auto/internal/pkg/process" "go.opentelemetry.io/auto/internal/pkg/process/binary" ) From 70d1d1bfae8cd16448cd8a07192d8bb51977ba82 Mon Sep 17 00:00:00 2001 From: Nikola Grcevski Date: Sun, 7 Apr 2024 15:44:38 -0400 Subject: [PATCH 09/14] Address PR feedback --- .../github.com/gin-gonic/gin/bpf/probe.bpf.c | 2 + .../bpf/net/http/client/bpf/probe.bpf.c | 45 +++++++++++++++---- .../bpf/net/http/client/probe.go | 36 ++++++++++----- internal/pkg/instrumentation/manager.go | 4 +- 4 files changed, 64 insertions(+), 23 deletions(-) diff --git a/internal/pkg/instrumentation/bpf/github.com/gin-gonic/gin/bpf/probe.bpf.c b/internal/pkg/instrumentation/bpf/github.com/gin-gonic/gin/bpf/probe.bpf.c index 7779e29d7..4067c4eca 100644 --- a/internal/pkg/instrumentation/bpf/github.com/gin-gonic/gin/bpf/probe.bpf.c +++ b/internal/pkg/instrumentation/bpf/github.com/gin-gonic/gin/bpf/probe.bpf.c @@ -58,6 +58,8 @@ int uprobe_GinEngine_ServeHTTP(struct pt_regs *ctx) { void *req_ptr = get_argument(ctx, request_pos); void *req_ctx_ptr = get_Go_context(ctx, 4, ctx_ptr_pos, false); + bpf_printk("Gin context %llx", req_ctx_ptr); + // Get method from request if (!get_go_string_from_user_ptr((void *)(req_ptr + method_ptr_pos), httpReq.method, sizeof(httpReq.method))) { bpf_printk("failed to get method from request"); diff --git a/internal/pkg/instrumentation/bpf/net/http/client/bpf/probe.bpf.c b/internal/pkg/instrumentation/bpf/net/http/client/bpf/probe.bpf.c index 117f67bef..5c85d962b 100644 --- a/internal/pkg/instrumentation/bpf/net/http/client/bpf/probe.bpf.c +++ b/internal/pkg/instrumentation/bpf/net/http/client/bpf/probe.bpf.c @@ -176,6 +176,7 @@ int uprobe_Transport_roundTrip_Returns(struct pt_regs *ctx) { return 0; } +#ifndef NO_HEADER_PROPAGATION SEC("uprobe/header_writeSubset") int uprobe_writeSubset(struct pt_regs *ctx) { u64 headers_pos = 1; @@ -196,32 +197,58 @@ int uprobe_writeSubset(struct pt_regs *ctx) { void *buf_ptr = 0; bpf_probe_read(&buf_ptr, sizeof(buf_ptr), (void *)(io_writer_ptr + io_writer_buf_ptr_pos)); // grab buf ptr if (!buf_ptr) { + bpf_printk("uprobe_writeSubset: Failed to get buf from io writer"); goto done; } s64 size = 0; - bpf_probe_read(&size, sizeof(s64), (void *)(io_writer_ptr + io_writer_buf_ptr_pos + 8)); // grab size + if (bpf_probe_read(&size, sizeof(s64), (void *)(io_writer_ptr + io_writer_buf_ptr_pos + 8))) { // grab size + bpf_printk("uprobe_writeSubset: Failed to get size from io writer"); + goto done; + } s64 len = 0; - bpf_probe_read(&len, sizeof(s64), (void *)(io_writer_ptr + io_writer_n_pos)); // grab len + if (bpf_probe_read(&len, sizeof(s64), (void *)(io_writer_ptr + io_writer_n_pos))) { // grab len + bpf_printk("uprobe_writeSubset: Failed to get len from io writer"); + goto done; + } -#ifndef NO_HEADER_PROPAGATION if (len < (size - W3C_VAL_LENGTH - W3C_KEY_LENGTH - 4)) { // 4 = strlen(":_") + strlen("\r\n") char key[W3C_KEY_LENGTH + 2] = "Traceparent: "; char end[2] = "\r\n"; - bpf_probe_write_user(buf_ptr + (len & 0x0ffff), key, sizeof(key)); + if (bpf_probe_write_user(buf_ptr + (len & 0x0ffff), key, sizeof(key))) { + bpf_printk("uprobe_writeSubset: Failed to write trace parent key in buffer"); + goto done; + } len += W3C_KEY_LENGTH + 2; - bpf_probe_write_user(buf_ptr + (len & 0x0ffff), tp, sizeof(tp)); + if (bpf_probe_write_user(buf_ptr + (len & 0x0ffff), tp, sizeof(tp))) { + bpf_printk("uprobe_writeSubset: Failed to write trace parent value in buffer"); + goto done; + } len += W3C_VAL_LENGTH; - bpf_probe_write_user(buf_ptr + (len & 0x0ffff), end, sizeof(end)); + if (bpf_probe_write_user(buf_ptr + (len & 0x0ffff), end, sizeof(end))) { + bpf_printk("uprobe_writeSubset: Failed to write new line in buffer"); + goto done; + } len += 2; - bpf_probe_write_user((void *)(io_writer_ptr + io_writer_n_pos), &len, sizeof(len)); + if (bpf_probe_write_user((void *)(io_writer_ptr + io_writer_n_pos), &len, sizeof(len))) { + bpf_printk("uprobe_writeSubset: Failed to change io writer n"); + goto done; + } } -#endif } } done: bpf_map_delete_elem(&http_headers, &headers_ptr); return 0; -} \ No newline at end of file +} +#else +// Not used at all, empty stub needed to ensure both versions of the bpf program are +// able to compile with bpf2go. The userspace code will avoid loading the probe if +// context propagation is not enabled. +SEC("uprobe/header_writeSubset") +int uprobe_writeSubset(struct pt_regs *ctx) { + return 0; +} +#endif diff --git a/internal/pkg/instrumentation/bpf/net/http/client/probe.go b/internal/pkg/instrumentation/bpf/net/http/client/probe.go index 61ce6c3d0..da929ff8e 100644 --- a/internal/pkg/instrumentation/bpf/net/http/client/probe.go +++ b/internal/pkg/instrumentation/bpf/net/http/client/probe.go @@ -50,6 +50,29 @@ func New(logger logr.Logger) probe.Probe { SpanKind: trace.SpanKindClient, InstrumentedPkg: pkg, } + + uprobes := []probe.Uprobe[bpfObjects]{ + { + Sym: "net/http.(*Transport).roundTrip", + Fn: uprobeRoundTrip, + }, + } + + // If the kernel supports context propagation, we enable the + // probe which writes the data in the outgoing buffer. + if utils.SupportsContextPropagation() { + uprobes = append(uprobes, + probe.Uprobe[bpfObjects]{ + Sym: "net/http.Header.writeSubset", + Fn: uprobeWriteSubset, + // We mark this probe as dependent on roundTrip, so we don't accidentally + // enable this bpf program, if the executable has compiled in writeSubset, + // but doesn't have any http roundTrip. + DependsOn: []string{"net/http.(*Transport).roundTrip"}, + }, + ) + } + return &probe.Base[bpfObjects, event]{ ID: id, Logger: logger.WithName(id.String()), @@ -101,18 +124,7 @@ func New(logger logr.Logger) probe.Probe { Val: structfield.NewID("std", "bufio", "Writer", "n"), }, }, - Uprobes: []probe.Uprobe[bpfObjects]{ - { - Sym: "net/http.(*Transport).roundTrip", - Fn: uprobeRoundTrip, - }, - { - Sym: "net/http.Header.writeSubset", - Fn: uprobeWriteSubset, - DependsOn: []string{"net/http.(*Transport).roundTrip"}, - }, - }, - + Uprobes: uprobes, ReaderFn: func(obj bpfObjects) (*perf.Reader, error) { return perf.NewReader(obj.Events, os.Getpagesize()) }, diff --git a/internal/pkg/instrumentation/manager.go b/internal/pkg/instrumentation/manager.go index a9601e797..aa0fbcd2d 100644 --- a/internal/pkg/instrumentation/manager.go +++ b/internal/pkg/instrumentation/manager.go @@ -106,13 +106,13 @@ func (m *Manager) FilterUnusedProbes(target *process.TargetDetails) { if len(s.DependsOn) != 0 { for _, depends := range s.DependsOn { if _, exists := existingFuncMap[depends]; exists { - funcsFound++ + funcsFound = true break } } } else { if _, exists := existingFuncMap[s.Symbol]; exists { - funcsFound++ + funcsFound = true break } } From b665043ca7150a34b03a595504b9ecd79b945d49 Mon Sep 17 00:00:00 2001 From: Nikola Grcevski Date: Sun, 7 Apr 2024 16:52:10 -0400 Subject: [PATCH 10/14] Update offsets --- internal/pkg/inject/offset_results.json | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/internal/pkg/inject/offset_results.json b/internal/pkg/inject/offset_results.json index e7bdd34eb..e2080bd52 100644 --- a/internal/pkg/inject/offset_results.json +++ b/internal/pkg/inject/offset_results.json @@ -1672,8 +1672,10 @@ "1.21.6", "1.21.7", "1.21.8", + "1.21.9", "1.22.0", - "1.22.1" + "1.22.1", + "1.22.2" ] } ] @@ -1829,8 +1831,10 @@ "1.21.6", "1.21.7", "1.21.8", + "1.21.9", "1.22.0", - "1.22.1" + "1.22.1", + "1.22.2" ] } ] From 13c95fc3b38a81b794d24a947679233ccbf45c88 Mon Sep 17 00:00:00 2001 From: Nikola Grcevski Date: Sun, 7 Apr 2024 16:59:30 -0400 Subject: [PATCH 11/14] simplify the changed code, since we don't need to count probes now --- internal/pkg/instrumentation/manager.go | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/internal/pkg/instrumentation/manager.go b/internal/pkg/instrumentation/manager.go index aa0fbcd2d..381ef10f3 100644 --- a/internal/pkg/instrumentation/manager.go +++ b/internal/pkg/instrumentation/manager.go @@ -103,14 +103,7 @@ func (m *Manager) FilterUnusedProbes(target *process.TargetDetails) { for name, inst := range m.probes { funcsFound := false for _, s := range inst.Manifest().Symbols { - if len(s.DependsOn) != 0 { - for _, depends := range s.DependsOn { - if _, exists := existingFuncMap[depends]; exists { - funcsFound = true - break - } - } - } else { + if len(s.DependsOn) == 0 { if _, exists := existingFuncMap[s.Symbol]; exists { funcsFound = true break From 9a8fa9f21016ba0b3a72ebc6c250f3b8e9fe83e0 Mon Sep 17 00:00:00 2001 From: Nikola Grcevski Date: Mon, 22 Apr 2024 14:12:07 -0400 Subject: [PATCH 12/14] atomic writes for traceparent --- .../bpf/net/http/client/bpf/probe.bpf.c | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/internal/pkg/instrumentation/bpf/net/http/client/bpf/probe.bpf.c b/internal/pkg/instrumentation/bpf/net/http/client/bpf/probe.bpf.c index 0591702b4..4f42c8a9a 100644 --- a/internal/pkg/instrumentation/bpf/net/http/client/bpf/probe.bpf.c +++ b/internal/pkg/instrumentation/bpf/net/http/client/bpf/probe.bpf.c @@ -212,23 +212,15 @@ int uprobe_writeSubset(struct pt_regs *ctx) { } if (len < (size - W3C_VAL_LENGTH - W3C_KEY_LENGTH - 4)) { // 4 = strlen(":_") + strlen("\r\n") - char key[W3C_KEY_LENGTH + 2] = "Traceparent: "; + char tp_str[W3C_KEY_LENGTH + 2 + W3C_VAL_LENGTH + 2] = "Traceparent: "; char end[2] = "\r\n"; - if (bpf_probe_write_user(buf_ptr + (len & 0x0ffff), key, sizeof(key))) { + __builtin_memcpy(&tp_str[W3C_KEY_LENGTH + 2], tp, sizeof(tp)); + __builtin_memcpy(&tp_str[W3C_KEY_LENGTH + 2 + W3C_VAL_LENGTH], end, sizeof(end)); + if (bpf_probe_write_user(buf_ptr + (len & 0x0ffff), tp_str, sizeof(tp_str))) { bpf_printk("uprobe_writeSubset: Failed to write trace parent key in buffer"); goto done; } - len += W3C_KEY_LENGTH + 2; - if (bpf_probe_write_user(buf_ptr + (len & 0x0ffff), tp, sizeof(tp))) { - bpf_printk("uprobe_writeSubset: Failed to write trace parent value in buffer"); - goto done; - } - len += W3C_VAL_LENGTH; - if (bpf_probe_write_user(buf_ptr + (len & 0x0ffff), end, sizeof(end))) { - bpf_printk("uprobe_writeSubset: Failed to write new line in buffer"); - goto done; - } - len += 2; + len += W3C_KEY_LENGTH + 2 + W3C_VAL_LENGTH + 2; if (bpf_probe_write_user((void *)(io_writer_ptr + io_writer_n_pos), &len, sizeof(len))) { bpf_printk("uprobe_writeSubset: Failed to change io writer n"); goto done; From 7157c928412b715eae6bac7b18853b84fd867a9f Mon Sep 17 00:00:00 2001 From: Nikola Grcevski Date: Mon, 22 Apr 2024 16:01:32 -0400 Subject: [PATCH 13/14] add dependents validation and tests --- internal/pkg/instrumentation/manager.go | 22 +++++ internal/pkg/instrumentation/manager_test.go | 84 ++++++++++++++++++++ 2 files changed, 106 insertions(+) diff --git a/internal/pkg/instrumentation/manager.go b/internal/pkg/instrumentation/manager.go index 355ee633d..33efd83c4 100644 --- a/internal/pkg/instrumentation/manager.go +++ b/internal/pkg/instrumentation/manager.go @@ -71,12 +71,34 @@ func NewManager(logger logr.Logger, otelController *opentelemetry.Controller, gl return m, nil } +func (m *Manager) validateProbeDependents(id probe.ID, symbols []probe.FunctionSymbol) error { + // Validate that dependent probes point to real standalone probes. + funcsMap := make(map[string]interface{}) + for _, s := range symbols { + funcsMap[s.Symbol] = nil + } + + for _, s := range symbols { + for _, d := range s.DependsOn { + if _, exists := funcsMap[d]; !exists { + return fmt.Errorf("library %s has declared a dependent function %s for probe %s which does not exist, aborting", id, d, s.Symbol) + } + } + } + + return nil +} + func (m *Manager) registerProbe(p probe.Probe) error { id := p.Manifest().Id if _, exists := m.probes[id]; exists { return fmt.Errorf("library %s registered twice, aborting", id) } + if err := m.validateProbeDependents(id, p.Manifest().Symbols); err != nil { + return err + } + m.probes[id] = p return nil } diff --git a/internal/pkg/instrumentation/manager_test.go b/internal/pkg/instrumentation/manager_test.go index 55973e4d8..21c9da328 100644 --- a/internal/pkg/instrumentation/manager_test.go +++ b/internal/pkg/instrumentation/manager_test.go @@ -23,6 +23,7 @@ import ( "github.com/hashicorp/go-version" "github.com/stretchr/testify/assert" + "go.opentelemetry.io/auto/internal/pkg/instrumentation/probe" "go.opentelemetry.io/auto/internal/pkg/process" "go.opentelemetry.io/auto/internal/pkg/process/binary" ) @@ -103,6 +104,89 @@ func TestProbeFiltering(t *testing.T) { }) } +func TestDependencyChecks(t *testing.T) { + m := fakeManager(t) + + t.Run("Dependent probes match", func(t *testing.T) { + syms := []probe.FunctionSymbol{ + { + Symbol: "A", + DependsOn: nil, + }, + { + Symbol: "B", + DependsOn: []string{"A"}, + }, + } + + assert.Nil(t, m.validateProbeDependents(probe.ID{InstrumentedPkg: "test"}, syms)) + }) + + t.Run("Second dependent missing", func(t *testing.T) { + syms := []probe.FunctionSymbol{ + { + Symbol: "A", + DependsOn: nil, + }, + { + Symbol: "B", + DependsOn: []string{"A", "C"}, + }, + } + + assert.NotNil(t, m.validateProbeDependents(probe.ID{InstrumentedPkg: "test"}, syms)) + }) + + t.Run("Second dependent present", func(t *testing.T) { + syms := []probe.FunctionSymbol{ + { + Symbol: "A", + DependsOn: nil, + }, + { + Symbol: "B", + DependsOn: []string{"A", "C"}, + }, + { + Symbol: "C", + DependsOn: []string{"A"}, + }, + } + + assert.Nil(t, m.validateProbeDependents(probe.ID{InstrumentedPkg: "test"}, syms)) + }) + + t.Run("Dependent wrong", func(t *testing.T) { + syms := []probe.FunctionSymbol{ + { + Symbol: "A", + DependsOn: nil, + }, + { + Symbol: "B", + DependsOn: []string{"A1"}, + }, + } + + assert.NotNil(t, m.validateProbeDependents(probe.ID{InstrumentedPkg: "test"}, syms)) + }) + + t.Run("Two probes without dependents", func(t *testing.T) { + syms := []probe.FunctionSymbol{ + { + Symbol: "A", + DependsOn: nil, + }, + { + Symbol: "B", + DependsOn: []string{}, + }, + } + + assert.Nil(t, m.validateProbeDependents(probe.ID{InstrumentedPkg: "test"}, syms)) + }) +} + func fakeManager(t *testing.T) *Manager { logger := stdr.New(log.New(os.Stderr, "", log.LstdFlags)) logger = logger.WithName("Instrumentation") From b1d95e817e712c1c7591a66b5176372dbe729602 Mon Sep 17 00:00:00 2001 From: Nikola Grcevski Date: Wed, 24 Apr 2024 14:01:17 -0400 Subject: [PATCH 14/14] apply pr feedback --- .../pkg/instrumentation/bpf/net/http/client/bpf/probe.bpf.c | 4 +++- internal/pkg/opentelemetry/controller.go | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/internal/pkg/instrumentation/bpf/net/http/client/bpf/probe.bpf.c b/internal/pkg/instrumentation/bpf/net/http/client/bpf/probe.bpf.c index 4f42c8a9a..25273c5f9 100644 --- a/internal/pkg/instrumentation/bpf/net/http/client/bpf/probe.bpf.c +++ b/internal/pkg/instrumentation/bpf/net/http/client/bpf/probe.bpf.c @@ -175,6 +175,8 @@ int uprobe_Transport_roundTrip_Returns(struct pt_regs *ctx) { } #ifndef NO_HEADER_PROPAGATION +// This instrumentation attaches uprobe to the following function: +// func (h Header) net/http.Header.writeSubset(w io.Writer, exclude map[string]bool, trace *httptrace.ClientTrace) error SEC("uprobe/header_writeSubset") int uprobe_writeSubset(struct pt_regs *ctx) { u64 headers_pos = 1; @@ -200,7 +202,7 @@ int uprobe_writeSubset(struct pt_regs *ctx) { } s64 size = 0; - if (bpf_probe_read(&size, sizeof(s64), (void *)(io_writer_ptr + io_writer_buf_ptr_pos + 8))) { // grab size + if (bpf_probe_read(&size, sizeof(s64), (void *)(io_writer_ptr + io_writer_buf_ptr_pos + offsetof(struct go_slice, cap)))) { // grab capacity bpf_printk("uprobe_writeSubset: Failed to get size from io writer"); goto done; } diff --git a/internal/pkg/opentelemetry/controller.go b/internal/pkg/opentelemetry/controller.go index d34dae24f..da678c955 100644 --- a/internal/pkg/opentelemetry/controller.go +++ b/internal/pkg/opentelemetry/controller.go @@ -52,7 +52,7 @@ func (c *Controller) getTracer(pkg string) trace.Tracer { // Trace creates a trace span for event. func (c *Controller) Trace(event *probe.Event) { for _, se := range event.SpanEvents { - c.logger.Info("got event", "kind", event.Kind.String(), "pkg", event.Package, "attrs", se.Attributes) + c.logger.Info("got event", "kind", event.Kind.String(), "pkg", event.Package, "attrs", se.Attributes, "traceID", se.SpanContext.TraceID().String(), "spanID", se.SpanContext.SpanID().String()) ctx := context.Background() if se.SpanContext == nil {