diff --git a/bpf/gotracer/go_nethttp.c b/bpf/gotracer/go_nethttp.c index 709d9926b9..301e21e0ae 100644 --- a/bpf/gotracer/go_nethttp.c +++ b/bpf/gotracer/go_nethttp.c @@ -27,8 +27,6 @@ #include #include #include -#include -#include #include @@ -37,9 +35,7 @@ #include -enum { k_tail_jsonrpc = 0 }; static const char traceparent[] = "traceparent: "; -static const char content_type[] = "content-type: "; typedef struct http_client_data { s64 content_length; @@ -70,13 +66,11 @@ typedef struct server_http_func_invocation { u64 content_length; u64 response_length; u64 status; - u64 body_addr; // pointer to the body buffer + u64 rpc_request_addr; // pointer to the jsonrpc Request tp_info_t tp; - unsigned char content_type[HTTP_CONTENT_TYPE_MAX_LEN]; u8 method[METHOD_MAX_LEN]; u8 path[PATH_MAX_LEN]; - u8 json_content_type; - u8 _pad[4]; + u8 _pad[5]; } server_http_func_invocation_t; struct { @@ -98,18 +92,6 @@ static __always_inline unsigned char *temp_header_mem() { return bpf_map_lookup_elem(&temp_header_mem_store, &zero); } -struct { - __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); - __type(key, u32); - __type(value, unsigned char[HTTP_BODY_MAX_LEN]); - __uint(max_entries, 1); -} temp_body_mem_store SEC(".maps"); - -static __always_inline unsigned char *temp_body_mem() { - const u32 zero = 0; - return bpf_map_lookup_elem(&temp_body_mem_store, &zero); -} - /* HTTP Server */ // This instrumentation attaches uprobe to the following function: @@ -151,15 +133,6 @@ int obi_uprobe_ServeHTTP(struct pt_regs *ctx) { // TODO: if context propagation is supported, overwrite the header value in the map with the // new span context and the same thread id. - // get content-type from readContinuedLineSlice - if (header_inv && header_inv->content_type[0]) { - bpf_dbg_printk("Found content type in ongoing request: %s", header_inv->content_type); - if (is_json_content_type((void *)header_inv->content_type, - sizeof(header_inv->content_type))) { - invocation.json_content_type = 1; - } - } - // Get method from Request.Method if (!read_go_str("method", req, @@ -311,13 +284,6 @@ static __always_inline void update_traceparent(server_http_func_invocation_t *in bpf_dbg_printk("Found traceparent in header %s", header_start); } -static __always_inline void update_content_type(server_http_func_invocation_t *inv, - const unsigned char *header_start) { - __builtin_memset(inv->content_type, 0, sizeof(inv->content_type)); - __builtin_memcpy(inv->content_type, header_start, sizeof(inv->content_type)); - bpf_dbg_printk("Found content-type in header %s", inv->content_type); -} - static __always_inline void handle_traceparent_header(server_http_func_invocation_t *inv, go_addr_key_t *g_key, unsigned char *traceparent_start) { @@ -330,18 +296,6 @@ static __always_inline void handle_traceparent_header(server_http_func_invocatio } } -static __always_inline void handle_content_type_header(server_http_func_invocation_t *inv, - go_addr_key_t *g_key, - unsigned char *content_type_start) { - if (inv) { - update_content_type(inv, content_type_start); - } else { - server_http_func_invocation_t minimal_inv = {}; - update_content_type(&minimal_inv, content_type_start); - bpf_map_update_elem(&ongoing_http_server_requests, g_key, &minimal_inv, BPF_ANY); - } -} - // Matches the header in the buffer and returns a pointer to the value part of the header. static __always_inline unsigned char *match_header( const unsigned char *buf, u32 safe_len, const char *header, u32 header_len, u32 value_len) { @@ -375,7 +329,6 @@ int obi_uprobe_readContinuedLineSliceReturns(struct pt_regs *ctx) { }; const u32 w3c_value_start = sizeof(traceparent) - 1; - const u32 content_type_value_start = sizeof(content_type) - 1; server_http_func_invocation_t *inv = bpf_map_lookup_elem(&ongoing_http_server_requests, &g_key); @@ -385,12 +338,6 @@ int obi_uprobe_readContinuedLineSliceReturns(struct pt_regs *ctx) { handle_traceparent_header(inv, &g_key, traceparent_start); } - unsigned char *content_type_start = match_header( - temp, safe_len, content_type, content_type_value_start, HTTP_CONTENT_TYPE_MAX_LEN); - if (content_type_start) { - handle_content_type_header(inv, &g_key, content_type_start); - } - return 0; } @@ -1299,99 +1246,68 @@ int obi_uprobe_netFdRead(struct pt_regs *ctx) { return 0; } -SEC("uprobe/bodyRead") -int obi_uprobe_bodyRead(struct pt_regs *ctx) { +SEC("uprobe/jsonrpcReadRequestHeader") +int obi_uprobe_jsonrpcReadRequestHeader(struct pt_regs *ctx) { void *goroutine_addr = GOROUTINE_PTR(ctx); - bpf_dbg_printk("=== uprobe/proc body read goroutine === "); + bpf_dbg_printk("=== uprobe/proc jsonrpc read request header goroutine %lx === ", + goroutine_addr); + go_addr_key_t g_key = {}; go_addr_key_from_id(&g_key, goroutine_addr); - // Get the address of the slice struct (p) - u64 body_addr = (u64)GO_PARAM2(ctx); - server_http_func_invocation_t *invocation = bpf_map_lookup_elem(&ongoing_http_server_requests, &g_key); if (!invocation) { return 0; } - invocation->body_addr = body_addr; + const u64 rpc_request_addr = (u64)GO_PARAM2(ctx); + bpf_dbg_printk("rpc_request_addr %llx", rpc_request_addr); + invocation->rpc_request_addr = rpc_request_addr; return 0; } -SEC("uprobe/bodyReadRet") -int obi_uprobe_bodyReadReturn(struct pt_regs *ctx) { - bpf_dbg_printk("=== uprobe/proc body read returns goroutine === "); +SEC("uprobe/jsonrpcReadRequestHeaderRet") +int obi_uprobe_jsonrpcReadRequestHeaderReturns(struct pt_regs *ctx) { void *goroutine_addr = GOROUTINE_PTR(ctx); + bpf_dbg_printk("=== uprobe/proc jsonrpc read request header return goroutine %lx === ", + goroutine_addr); + go_addr_key_t g_key = {}; go_addr_key_from_id(&g_key, goroutine_addr); - u64 n = (u64)GO_PARAM1(ctx); - bpf_dbg_printk("n is %llu", n); - server_http_func_invocation_t *invocation = bpf_map_lookup_elem(&ongoing_http_server_requests, &g_key); - if (!invocation) { - return 0; - } - if (n <= 0 || !invocation->body_addr) { - return 0; - } - // json_content-type is set in invocation in ServeHTTP - if (invocation->json_content_type != 1) { - bpf_dbg_printk("content type is not json, skipping"); - return 0; - } - unsigned char *body_buf = temp_body_mem(); - if (!body_buf) { + if (!invocation || !invocation->rpc_request_addr) { return 0; } - const u32 safe_len = n > HTTP_BODY_MAX_LEN ? HTTP_BODY_MAX_LEN : n; - if (!read_go_str_n("http body", (void *)invocation->body_addr, n, body_buf, safe_len)) { - bpf_dbg_printk("failed to read body, n=%llu, body_addr=%llx", n, invocation->body_addr); - return 0; - } - // bpf_dbg_printk("body is %s", body_buf); - bpf_tail_call(ctx, &jsonrpc_jump_table, k_tail_jsonrpc); - return 0; -} + off_table_t *ot = get_offsets_table(); -//k_tail_jsonrpc -SEC("uprobe/readJsonrpcMethod") -int obi_read_jsonrpc_method(struct pt_regs *ctx) { - bpf_dbg_printk("=== uprobe/proc read jsonrpc method === "); - void *goroutine_addr = GOROUTINE_PTR(ctx); - go_addr_key_t g_key = {}; - go_addr_key_from_id(&g_key, goroutine_addr); + const u64 rpc_request_addr = invocation->rpc_request_addr; - server_http_func_invocation_t *invocation = - bpf_map_lookup_elem(&ongoing_http_server_requests, &g_key); - if (!invocation) { - bpf_dbg_printk("can't find invocation info for server call"); + bpf_dbg_printk("rpc_request_addr %llx", rpc_request_addr); + + const u64 method_len = peek_go_str_len( + "JSON-RPC method", + (void *)rpc_request_addr, + go_offset_of(ot, (go_offset){.v = _jsonrpc_request_header_service_method_pos})); + + if (method_len == 0) { return 0; } - // tail call is guaranteed to run on the same CPU as its caller - // so we can shared the same buffer via a per-CPU map - unsigned char *body_buf = temp_body_mem(); - if (!body_buf) { + if (!read_go_str("JSON-RPC method", + (void *)rpc_request_addr, + go_offset_of(ot, (go_offset){.v = _jsonrpc_request_header_service_method_pos}), + invocation->method, + METHOD_MAX_LEN)) { + bpf_dbg_printk("Failed to read JSON-RPC method from %llx", rpc_request_addr); return 0; } - if (is_jsonrpc2_body((const unsigned char *)body_buf, HTTP_BODY_MAX_LEN)) { - unsigned char method_buf[JSONRPC_METHOD_BUF_SIZE] = {}; - u32 method_len = extract_jsonrpc2_method( - (const unsigned char *)body_buf, HTTP_BODY_MAX_LEN, method_buf, sizeof(method_buf)); - if (method_len > 0) { - bpf_dbg_printk("JSON-RPC method: %s", method_buf); - read_go_str_n("JSON-RPC method", - (void *)method_buf, - method_len, - invocation->method, - sizeof(invocation->method)); - } - } + bpf_dbg_printk("read jsonrpc method %s", invocation->method); + return 0; } diff --git a/bpf/gotracer/go_offsets.h b/bpf/gotracer/go_offsets.h index 308eeafa81..e0fdaa0689 100644 --- a/bpf/gotracer/go_offsets.h +++ b/bpf/gotracer/go_offsets.h @@ -75,6 +75,8 @@ typedef enum { _tracer_delegate_pos, _tracer_attribute_opt_off, _error_string_off, + // go jsonrpc + _jsonrpc_request_header_service_method_pos, _last_go_offset, } go_offset_const; diff --git a/bpf/gotracer/go_str.h b/bpf/gotracer/go_str.h index 658441599f..853ae0282b 100644 --- a/bpf/gotracer/go_str.h +++ b/bpf/gotracer/go_str.h @@ -59,3 +59,12 @@ read_go_str(char *name, void *base_ptr, u8 offset, void *field, u64 max_size) { return 1; } + +static __always_inline u64 peek_go_str_len(const char *name, const void *base_ptr, u8 offset) { + u64 len = 0; + if (bpf_probe_read(&len, sizeof(len), (const void *)(base_ptr + (offset + 8))) != 0) { + bpf_dbg_printk("can't read len for %s", name); + return 0; + } + return len; +} diff --git a/bpf/gotracer/maps/jsonrpc_jump_table.h b/bpf/gotracer/maps/jsonrpc_jump_table.h deleted file mode 100644 index 02b8e88f75..0000000000 --- a/bpf/gotracer/maps/jsonrpc_jump_table.h +++ /dev/null @@ -1,11 +0,0 @@ -#pragma once - -#include -#include - -struct { - __uint(type, BPF_MAP_TYPE_PROG_ARRAY); - __type(key, u32); - __type(value, u32); - __uint(max_entries, 1); -} jsonrpc_jump_table SEC(".maps"); diff --git a/bpf/gotracer/protocol_jsonrpc.h b/bpf/gotracer/protocol_jsonrpc.h deleted file mode 100644 index 9a8537a882..0000000000 --- a/bpf/gotracer/protocol_jsonrpc.h +++ /dev/null @@ -1,162 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include - -static const char k_jsonrpc_key[] = "\"jsonrpc\""; -static const u32 k_jsonrpc_key_len = sizeof(k_jsonrpc_key) - 1; -static const char k_jsonrpc_val[] = "\"2.0\""; -static const u32 k_jsonrpc_val_len = sizeof(k_jsonrpc_val) - 1; -static const char k_application_json[] = "application/json"; -static const u32 k_application_json_len = sizeof(k_application_json) - 1; -static const char k_method_key[] = "\"method\""; -static const u32 k_method_key_len = sizeof(k_method_key) - 1; - -enum { JSON_MAX_STRING_LEN = 16, JSONRPC_METHOD_BUF_SIZE = 16 }; - -// should match application/json, application/json-rpc, application/jsonrequest -// listed in https://www.jsonrpc.org/historical/json-rpc-over-http.html -static __always_inline u8 is_json_content_type(const char *c, u32 len) { - if (len < k_application_json_len) { - return 0; - } - // Check for "application/json" at the start - if (c[0] == 'a' && c[1] == 'p' && c[2] == 'p' && c[3] == 'l' && c[4] == 'i' && c[5] == 'c' && - c[6] == 'a' && c[7] == 't' && c[8] == 'i' && c[9] == 'o' && c[10] == 'n' && c[11] == '/' && - c[12] == 'j' && c[13] == 's' && c[14] == 'o' && c[15] == 'n') { - return 1; - } - return 0; -} - -// ref: https://en.cppreference.com/w/c/string/byte/isspace -static __always_inline u8 bpf_isspace(char c) { - return (c == ' ' || c == '\f' || c == '\n' || c == '\r' || c == '\t' || c == '\v'); -} - -// Returns the offset of the next JSON value after skipping whitespace and colon. -// If not found, returns body_len. -static __always_inline u32 json_value_offset(const unsigned char *body, - u32 body_len, - u32 start_pos) { - u32 pos = start_pos; - while (pos < body_len && (bpf_isspace(body[pos]) || body[pos] == ':')) { - pos++; - } - return pos; -} - -// Returns the position of the first occurrence of a string in a JSON body. -// If not found, returns INVALID_POS. -static __always_inline u32 json_str_value(const unsigned char *body, - u32 body_len, - const unsigned char *str, - u32 str_len) { - return bpf_memstr(body, body_len, str, str_len); -} - -// Compares a JSON value at start with a given value. -static __always_inline bool json_value_eq(const char *start, const char *val, u32 val_len) { - - return stricmp(start, val, val_len); -} - -// Extracts a JSON string value starting at a given position. -// Copies up to buf_len-1 bytes into buf, null-terminated. -// Returns the number of bytes copied (not including null terminator), or 0 on error. -static __always_inline u32 extract_json_string( - const unsigned char *body, u32 body_len, u32 value_start, unsigned char *buf, u32 buf_len) { - if (value_start >= body_len || buf_len == 0) { - return 0; - } - - if (body[value_start] != '"') { - return 0; - } - - const u32 str_start = value_start + 1; - u32 value_end = str_start; - while (value_end < body_len && body[value_end] != '"') { - value_end++; - } - const u32 value_len = value_end - str_start; - if (value_len == 0) { - return 0; - } - - const u32 copy_len = value_len < (buf_len - 1) ? value_len : (buf_len - 1); - - for (u32 i = 0; i < buf_len; i++) { - if (i >= copy_len) { - break; - } - buf[i] = body[str_start + i]; - } - buf[copy_len] = '\0'; - return copy_len; -} - -// Looks for '"jsonrpc":"2.0"' -static __always_inline u32 is_jsonrpc2_body(const unsigned char *body, u32 body_len) { - u32 key_pos = - json_str_value(body, body_len, (const unsigned char *)k_jsonrpc_key, k_jsonrpc_key_len); - if (key_pos == INVALID_POS) { - return 0; - } - - bpf_dbg_printk("Found JSON-RPC 2.0 key"); - - u32 val_search_start = key_pos + k_jsonrpc_key_len; - if (val_search_start >= body_len) { - return 0; - } - - val_search_start = json_value_offset(body, body_len, val_search_start); - if (val_search_start >= body_len) { - return 0; - } - - if (val_search_start + k_jsonrpc_val_len >= body_len) { - return 0; - } - - if (!json_value_eq((const char *)(body + val_search_start), - (const char *)k_jsonrpc_val, - k_jsonrpc_val_len)) { - return 0; - } - - bpf_dbg_printk("Found JSON-RPC 2.0 value"); - - return 1; // JSON-RPC 2.0 detected -} - -// Extracts the value of the "method" key from a JSON-RPC 2.0 body. -// Returns the length of the method value, or 0 if not found or error. -// method_buf must be at least method_buf_len bytes. -static __always_inline u32 extract_jsonrpc2_method(const unsigned char *body, - u32 body_len, - unsigned char *method_buf, - u32 method_buf_len) { - u32 key_pos = - json_str_value(body, body_len, (const unsigned char *)k_method_key, k_method_key_len); - if (key_pos == INVALID_POS) { - return 0; - } - - bpf_dbg_printk("Found JSON-RPC method key"); - - u32 val_search_start = key_pos + k_method_key_len; - if (val_search_start >= body_len) { - return 0; - } - - val_search_start = json_value_offset(body, body_len, val_search_start); - if (val_search_start >= body_len) { - return 0; - } - return extract_json_string(body, body_len, val_search_start, method_buf, method_buf_len); -} \ No newline at end of file diff --git a/configs/offsets/std_inspect.go b/configs/offsets/std_inspect.go index 058d905af5..940ee92572 100644 --- a/configs/offsets/std_inspect.go +++ b/configs/offsets/std_inspect.go @@ -6,12 +6,50 @@ package main import ( "context" "fmt" + "io" "net/http" + "net/rpc" + "net/rpc/jsonrpc" "os" ) // This program is used to generate an executable that can be inspected by the go-offsets-tracker tool +// Args defines the arguments for the RPC methods. +type Args struct { + A, B int +} + +// Arith provides methods for arithmetic operations. +type Arith struct{} + +// Multiply multiplies two numbers and returns the result. +func (t *Arith) Multiply(args *Args, reply *int) error { + *reply = args.A * args.B + return nil +} + +type ReadWriteCloserWrapper struct { + io.Reader + io.Writer +} + +// Close is a no-op to satisfy the io.ReadWriteCloser interface. +func (w *ReadWriteCloserWrapper) Close() error { + return nil +} + +func jsonrpcHandler(writer http.ResponseWriter, request *http.Request) { + if request.Method != http.MethodPost { + http.Error(writer, "Only POST method is allowed", http.StatusMethodNotAllowed) + return + } + // Wrap the request body and response writer in a ReadWriteCloser. + conn := &ReadWriteCloserWrapper{Reader: request.Body, Writer: writer} + // Serve the request using JSON-RPC codec. + rpc.ServeCodec(jsonrpc.NewServerCodec(conn)) +} + func regularGetRequest(ctx context.Context, url string) error { req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) if err != nil { @@ -43,4 +81,11 @@ func main() { if err != nil { os.Exit(1) } + // Register the Arith service. + arith := new(Arith) + rpc.Register(arith) + err = http.ListenAndServe(":8080", http.HandlerFunc(jsonrpcHandler)) + if err != nil { + os.Exit(1) + } } diff --git a/configs/offsets/tracker_input.json b/configs/offsets/tracker_input.json index 6d23c79fd4..506f30f999 100644 --- a/configs/offsets/tracker_input.json +++ b/configs/offsets/tracker_input.json @@ -63,6 +63,9 @@ ], "net/http.http2MetaHeadersFrame": [ "Fields" + ], + "net/rpc.Request": [ + "ServiceMethod" ] } }, diff --git a/pkg/components/ebpf/gotracer/gotracer.go b/pkg/components/ebpf/gotracer/gotracer.go index 40b0fee65e..8080dbff64 100644 --- a/pkg/components/ebpf/gotracer/gotracer.go +++ b/pkg/components/ebpf/gotracer/gotracer.go @@ -88,20 +88,6 @@ func (p *Tracer) Load() (*ebpf.CollectionSpec, error) { } func (p *Tracer) SetupTailCalls() { - for _, tc := range []struct { - index int - prog *ebpf.Program - }{ - { - index: 0, - prog: p.bpfObjects.ObiReadJsonrpcMethod, - }, - } { - err := p.bpfObjects.JsonrpcJumpTable.Update(uint32(tc.index), uint32(tc.prog.FD()), ebpf.UpdateAny) - if err != nil { - p.log.Error("error loading info tail call jump table", "error", err) - } - } } func (p *Tracer) Constants() map[string]any { @@ -192,6 +178,8 @@ func (p *Tracer) RegisterOffsets(fileInfo *exec.FileInfo, offsets *goexec.Offset goexec.GrpcClientStreamStream, // go manual spans goexec.GoTracerDelegatePos, + // go jsonrpc + goexec.GoJsonrpcRequestHeaderServiceMethodPos, } { if val, ok := offsets.Field[field].(uint64); ok { offTable.Table[field] = val @@ -250,9 +238,10 @@ func (p *Tracer) GoProbes() map[string][]*ebpfcommon.ProbeDesc { Start: p.bpfObjects.ObiUprobeReadRequestStart, End: p.bpfObjects.ObiUprobeReadRequestReturns, }}, - "net/http.(*body).Read": {{ - Start: p.bpfObjects.ObiUprobeBodyRead, - End: p.bpfObjects.ObiUprobeBodyReadReturn, + // Go net/rpc/jsonrpc + "net/rpc/jsonrpc.(*serverCodec).ReadRequestHeader": {{ + Start: p.bpfObjects.ObiUprobeJsonrpcReadRequestHeader, + End: p.bpfObjects.ObiUprobeJsonrpcReadRequestHeaderReturns, }}, "net/textproto.(*Reader).readContinuedLineSlice": {{ End: p.bpfObjects.ObiUprobeReadContinuedLineSliceReturns, diff --git a/pkg/components/goexec/offsets.json b/pkg/components/goexec/offsets.json index 16d479f390..18e895f5ac 100755 --- a/pkg/components/goexec/offsets.json +++ b/pkg/components/goexec/offsets.json @@ -924,6 +924,20 @@ ] } }, + "net/rpc.Request": { + "ServiceMethod": { + "versions": { + "oldest": "1.17.0", + "newest": "1.24.5" + }, + "offsets": [ + { + "offset": 0, + "since": "1.17.0" + } + ] + } + }, "net/url.URL": { "Host": { "versions": { diff --git a/pkg/components/goexec/structmembers.go b/pkg/components/goexec/structmembers.go index 901737eff0..77aaf7e6f5 100644 --- a/pkg/components/goexec/structmembers.go +++ b/pkg/components/goexec/structmembers.go @@ -98,6 +98,8 @@ const ( GoTracerDelegatePos GoTracerAttributeOptOffset GoErrorStringOffset + // go jsonrpc + GoJsonrpcRequestHeaderServiceMethodPos ) //go:embed offsets.json @@ -290,6 +292,12 @@ var structMembers = map[string]structInfo{ "tlsState": CTlsPos, }, }, + "net/rpc.Request": { + lib: "go", + fields: map[string]GoOffset{ + "ServiceMethod": GoJsonrpcRequestHeaderServiceMethodPos, + }, + }, "google.golang.org/grpc/internal/transport.bufWriter": { lib: "google.golang.org/grpc", fields: map[string]GoOffset{ diff --git a/test/integration/traces_test.go b/test/integration/traces_test.go index 0a8aa13cf5..a869c92f8d 100644 --- a/test/integration/traces_test.go +++ b/test/integration/traces_test.go @@ -769,7 +769,6 @@ func testNestedHTTPTracesKProbes(t *testing.T) { ) assert.Empty(t, sd, sd.String()) - /* FIXME flaky // Check the information of the go jsonrpc parent span res = trace.FindByOperationName("Arith.T /jsonrpc", "server") require.Len(t, res, 1) @@ -789,7 +788,6 @@ func testNestedHTTPTracesKProbes(t *testing.T) { jaeger.Tag{Key: "span.kind", Type: "string", Value: "server"}, ) assert.Empty(t, sd, sd.String()) - */ // Check the information of the python parent span res = trace.FindByOperationName("GET /tracemetoo", "server")