From cd7ab1371a013c37e31a17184b65af81e70fa106 Mon Sep 17 00:00:00 2001 From: titaneric Date: Sat, 14 Jun 2025 13:27:54 +0000 Subject: [PATCH 01/40] apply patch from beyla --- bpf/common/common.h | 4 + bpf/common/tc_common.h | 27 +++- bpf/gotracer/go_nethttp.c | 151 ++++++++++++++++++++--- bpf/gotracer/protocol_jsonrpc.h | 136 ++++++++++++++++++++ pkg/components/ebpf/gotracer/gotracer.go | 4 + 5 files changed, 302 insertions(+), 20 deletions(-) create mode 100644 bpf/gotracer/protocol_jsonrpc.h diff --git a/bpf/common/common.h b/bpf/common/common.h index 105d812d8f..5ef8fce924 100644 --- a/bpf/common/common.h +++ b/bpf/common/common.h @@ -30,6 +30,10 @@ #define MAX_TOPIC_NAME_LEN 64 #define HOST_MAX_LEN 100 #define SCHEME_MAX_LEN 10 +#define HTTP_BODY_MAX_LEN 64 +#define HTTP_HEADER_MAX_LEN 100 +#define CONTENT_TYPE_KEY_LEN 12 // "content-type" length +#define HTTP_CONTENT_TYPE_MAX_LEN 16 // Trace of an HTTP call invocation. It is instantiated by the return uprobe and forwarded to the // user space through the events ringbuffer. diff --git a/bpf/common/tc_common.h b/bpf/common/tc_common.h index 57cc2a2f63..2e18bf75ef 100644 --- a/bpf/common/tc_common.h +++ b/bpf/common/tc_common.h @@ -3,7 +3,7 @@ #include #include -enum { MAX_INLINE_LEN = 0x3ff }; +enum { MAX_INLINE_LEN = 0x3ff, MAX_NEEDLE_LEN = 16 }; const char TP[] = "Traceparent: 00-00000000000000000000000000000000-0000000000000000-01\r\n"; const u32 EXTEND_SIZE = sizeof(TP) - 1; @@ -131,3 +131,28 @@ static __always_inline u32 sk_msg_local_port(struct sk_msg_md *ctx) { return data; } + +// find the needle in the haystack, return the position of the first occurrence, return -1 if not found +static __always_inline u32 bpf_memstr(const char *haystack, + int haystack_len, + const char *needle, + int needle_len) { + if (needle_len == 0 || haystack_len < needle_len) + return INVALID_POS; + for (int i = 0; i <= haystack_len - needle_len; i++) { + int found = 1; +#pragma unroll + // max needle length + for (int j = 0; j < MAX_NEEDLE_LEN; j++) { + if (j >= needle_len) + break; + if (haystack[i + j] != needle[j]) { + found = 0; + break; + } + } + if (found) + return (u32)i; + } + return INVALID_POS; +} \ No newline at end of file diff --git a/bpf/gotracer/go_nethttp.c b/bpf/gotracer/go_nethttp.c index 2df942ef12..acd120a9b8 100644 --- a/bpf/gotracer/go_nethttp.c +++ b/bpf/gotracer/go_nethttp.c @@ -26,6 +26,7 @@ #include #include #include +#include #include @@ -67,6 +68,8 @@ typedef struct server_http_func_invocation { unsigned char method[METHOD_MAX_LEN]; unsigned char path[PATH_MAX_LEN]; u8 _pad[5]; + u64 body_addr; // pointer to the body buffer + u8 content_type[HTTP_CONTENT_TYPE_MAX_LEN]; } server_http_func_invocation_t; struct { @@ -93,12 +96,12 @@ int beyla_uprobe_ServeHTTP(struct pt_regs *ctx) { off_table_t *ot = get_offsets_table(); - // Lookup any traceparent information setup for us by readContinuedLineSlice - server_http_func_invocation_t *tp_inv = + // Lookup any header information setup for us by readContinuedLineSlice + server_http_func_invocation_t *header_inv = bpf_map_lookup_elem(&ongoing_http_server_requests, &g_key); tp_info_t *decoded_tp = 0; - if (tp_inv && valid_trace(tp_inv->tp.trace_id)) { - decoded_tp = &tp_inv->tp; + if (header_inv && valid_trace(header_inv->tp.trace_id)) { + decoded_tp = &header_inv->tp; } server_http_func_invocation_t invocation = { @@ -117,6 +120,14 @@ int beyla_uprobe_ServeHTTP(struct pt_regs *ctx) { // TODO: if context propagation is supported, overwrite the header value in the map with the // new span context and the same thread id. + // get content-type from readContinuedLineSlice + if (header_inv && header_inv->content_type[0]) { + bpf_dbg_printk("Found content type in ongoing request: %s", header_inv->content_type); + __builtin_memcpy(invocation.content_type, + header_inv->content_type, + sizeof(header_inv->content_type)); + } + // Get method from Request.Method if (!read_go_str("method", req, @@ -262,6 +273,19 @@ int beyla_uprobe_http2Server_processHeaders(struct pt_regs *ctx) { return 0; } +static __always_inline void update_traceparent(server_http_func_invocation_t *inv, + u8 *header_start) { + decode_go_traceparent(header_start, inv->tp.trace_id, inv->tp.parent_id, &inv->tp.flags); + bpf_dbg_printk("Found traceparent in header %s", header_start); +} + +static __always_inline void update_content_type(server_http_func_invocation_t *inv, + u8 *header_start) { + __builtin_memset(inv->content_type, 0, sizeof(inv->content_type)); + __builtin_memcpy(inv->content_type, header_start, sizeof(inv->content_type)); + bpf_dbg_printk("Found content-type in header %s", inv->content_type); +} + SEC("uprobe/readContinuedLineSlice") int beyla_uprobe_readContinuedLineSliceReturns(struct pt_regs *ctx) { bpf_dbg_printk("=== uprobe/proc readContinuedLineSlice returns === "); @@ -270,22 +294,45 @@ int beyla_uprobe_readContinuedLineSliceReturns(struct pt_regs *ctx) { u64 len = (u64)GO_PARAM2(ctx); const unsigned char *buf = (const unsigned char *)GO_PARAM1(ctx); - if (len >= (W3C_KEY_LENGTH + W3C_VAL_LENGTH + 2)) { - unsigned char temp[W3C_KEY_LENGTH + W3C_VAL_LENGTH + 2]; - bpf_probe_read(temp, sizeof(temp), buf); - bpf_dbg_printk("goroutine_addr %lx", goroutine_addr); - go_addr_key_t g_key = {}; - go_addr_key_from_id(&g_key, goroutine_addr); + u8 temp[HTTP_HEADER_MAX_LEN]; + if (len > sizeof(temp)) + len = sizeof(temp); + bpf_probe_read(temp, len, buf); - connection_info_t *existing = bpf_map_lookup_elem(&ongoing_server_connections, &g_key); - if (existing) { - if (!bpf_memicmp((const char *)temp, "traceparent: ", W3C_KEY_LENGTH + 2)) { - server_http_func_invocation_t inv = {}; - decode_go_traceparent( - temp + W3C_KEY_LENGTH + 2, inv.tp.trace_id, inv.tp.parent_id, &inv.tp.flags); - bpf_dbg_printk("Found traceparent in header %s", temp); - bpf_map_update_elem(&ongoing_http_server_requests, &g_key, &inv, BPF_ANY); - } + bpf_dbg_printk("goroutine_addr %lx", goroutine_addr); + go_addr_key_t g_key = {}; + go_addr_key_from_id(&g_key, goroutine_addr); + + int w3c_value_start = W3C_KEY_LENGTH + 2; // "traceparent: " + int w3c_header_length = w3c_value_start + W3C_VAL_LENGTH; + int content_type_value_start = CONTENT_TYPE_KEY_LEN + 2; // "content-type: " + int content_type_header_length = content_type_value_start + HTTP_CONTENT_TYPE_MAX_LEN; + + connection_info_t *existing = bpf_map_lookup_elem(&ongoing_server_connections, &g_key); + if (!existing) { + return 0; + } + server_http_func_invocation_t *inv = bpf_map_lookup_elem(&ongoing_http_server_requests, &g_key); + + if (len >= w3c_header_length && + !bpf_memicmp((const char *)temp, "traceparent: ", w3c_value_start)) { + u8 *traceparent_start = temp + w3c_value_start; + if (inv) { + update_traceparent(inv, traceparent_start); + } else { + server_http_func_invocation_t minimal_inv = {}; + update_traceparent(&minimal_inv, traceparent_start); + bpf_map_update_elem(&ongoing_http_server_requests, &g_key, &minimal_inv, BPF_ANY); + } + } else if (len >= content_type_header_length && + !bpf_memicmp((const char *)temp, "content-type: ", content_type_value_start)) { + u8 *content_type_start = temp + content_type_value_start; + if (inv) { + update_content_type(inv, content_type_start); + } else { + server_http_func_invocation_t minimal_inv = {}; + update_content_type(&minimal_inv, content_type_start); + bpf_map_update_elem(&ongoing_http_server_requests, &g_key, &minimal_inv, BPF_ANY); } } @@ -1181,6 +1228,72 @@ int beyla_uprobe_netFdRead(struct pt_regs *ctx) { return 0; } +SEC("uprobe/bodyRead") +int beyla_uprobe_bodyRead(struct pt_regs *ctx) { + void *goroutine_addr = GOROUTINE_PTR(ctx); + bpf_dbg_printk("=== uprobe/proc body read goroutine === "); + go_addr_key_t g_key = {}; + go_addr_key_from_id(&g_key, goroutine_addr); + + // Get the address of the slice struct (p) + u64 body_addr = (u64)GO_PARAM2(ctx); + + server_http_func_invocation_t *invocation = + bpf_map_lookup_elem(&ongoing_http_server_requests, &g_key); + if (!invocation) { + bpf_dbg_printk("can't find invocation info for server call"); + return 0; + } + invocation->body_addr = body_addr; + + return 0; +} + +SEC("uprobe/bodyReadRet") +int beyla_uprobe_bodyReadReturn(struct pt_regs *ctx) { + void *goroutine_addr = GOROUTINE_PTR(ctx); + bpf_dbg_printk("=== uprobe/proc body read returns goroutine === "); + go_addr_key_t g_key = {}; + go_addr_key_from_id(&g_key, goroutine_addr); + + u64 n = (u64)GO_PARAM1(ctx); + + server_http_func_invocation_t *invocation = + bpf_map_lookup_elem(&ongoing_http_server_requests, &g_key); + if (!invocation) { + bpf_dbg_printk("can't find invocation info for server call"); + return 0; + } + // content-type is set in invocation in ServeHTTP + bpf_dbg_printk("n is %d", n); + bpf_dbg_printk("content type is %s", invocation->content_type); + + char body_buf[HTTP_BODY_MAX_LEN] = {}; + if (n > 0 && invocation->body_addr) { + if (is_json_content_type((void *)invocation->content_type, + sizeof(invocation->content_type))) { + if (read_go_str_n( + "http body", (void *)invocation->body_addr, n, body_buf, sizeof(body_buf))) { + bpf_dbg_printk("body is %s", body_buf); + if (is_jsonrpc2_body(body_buf, sizeof(body_buf))) { + char method_buf[JSONRPC_METHOD_BUF_SIZE] = {}; + u32 method_len = + extract_jsonrpc2_method(body_buf, sizeof(body_buf), method_buf); + if (method_len > 0) { + bpf_dbg_printk("JSON-RPC method: %s", method_buf); + read_go_str_n("JSON-RPC method", + (void *)method_buf, + method_len, + invocation->method, + sizeof(invocation->method)); + } + } + } + } + } + return 0; +} + SEC("uprobe/connServeRet") int beyla_uprobe_connServeRet(struct pt_regs *ctx) { bpf_dbg_printk("=== uprobe/proc http conn serve ret === "); diff --git a/bpf/gotracer/protocol_jsonrpc.h b/bpf/gotracer/protocol_jsonrpc.h new file mode 100644 index 0000000000..13dd832100 --- /dev/null +++ b/bpf/gotracer/protocol_jsonrpc.h @@ -0,0 +1,136 @@ +#include +#include +#include + +static const char k_jsonrpc_key[] = "\"jsonrpc\""; +static const u32 k_jsonrpc_key_len = sizeof(k_jsonrpc_key) - 1; +static const char k_jsonrpc_val[] = "\"2.0\""; +static const u32 k_jsonrpc_val_len = sizeof(k_jsonrpc_val) - 1; +static const char k_application_json[] = "application/json"; +static const u32 k_application_json_len = sizeof(k_application_json) - 1; +static const char k_method_key[] = "\"method\""; +static const u32 k_method_key_len = sizeof(k_method_key) - 1; + +enum { JSONRPC_METHOD_BUF_SIZE = 16 }; + +// should match application/json, application/json-rpc, application/jsonrequest +// listed in https://www.jsonrpc.org/historical/json-rpc-over-http.html +static __always_inline u8 is_json_content_type(const char *c, u32 len) { + if (len < k_application_json_len) { + return 0; + } + // Check for "application/json" at the start + if (c[0] == 'a' && c[1] == 'p' && c[2] == 'p' && c[3] == 'l' && c[4] == 'i' && c[5] == 'c' && + c[6] == 'a' && c[7] == 't' && c[8] == 'i' && c[9] == 'o' && c[10] == 'n' && c[11] == '/' && + c[12] == 'j' && c[13] == 's' && c[14] == 'o' && c[15] == 'n') { + return 1; + } + return 0; +} + +// ref: https://en.cppreference.com/w/c/string/byte/isspace +static __always_inline u8 bpf_isspace(char c) { + return (c == ' ' || c == '\f' || c == '\n' || c == '\r' || c == '\t' || c == '\v'); +} + +// Returns the offset of the next JSON value after skipping whitespace and colon. +// If not found, returns body_len. +static __always_inline u32 json_value_offset(const char *body, u32 body_len, u32 start_pos) { + u32 pos = start_pos; + while (pos < body_len && (bpf_isspace(body[pos]) || body[pos] == ':')) { + pos++; + } + return pos; +} + +// Returns the position of the first occurrence of a string in a JSON body. +// If not found, returns INVALID_POS. +static __always_inline u32 json_str_value(const char *body, + u32 body_len, + const char *str, + u32 str_len) { + return bpf_memstr((const char *)body, body_len, (const char *)str, str_len); +} + +// Returns the end position (index of closing quote) of a JSON string value. +// If not found, returns body_len. +static __always_inline u32 json_str_value_end(const char *body, u32 body_len, u32 value_start) { + // find_first_pos_of expects unsigned char*, so cast accordingly + return value_start + find_first_pos_of((unsigned char *)(body + value_start), + (unsigned char *)(body + body_len), + '"'); +} + +/** + * Copies a JSON string value from body[value_start..value_end) into dest_buf. + * Ensures null-termination and does not exceed dest_buf_size. + * Returns the number of bytes copied (excluding null terminator), or 0 on error. + */ +static __always_inline u32 copy_json_string_value( + const char *body, u32 value_start, u32 value_end, char *dest_buf, u32 dest_buf_size) { + u32 value_len = value_end - value_start; + if (value_len <= 0) + return 0; + if (value_len >= dest_buf_size) + value_len = dest_buf_size - 1; // leave space for null terminator + +#pragma unroll + for (u32 i = 0; i < dest_buf_size; i++) { + if (i >= value_len) + break; + dest_buf[i] = body[value_start + i]; + } + dest_buf[value_len] = '\0'; + return value_len; +} + +// Looks for '"jsonrpc":"2.0"' +static __always_inline u32 is_jsonrpc2_body(const char *body, u32 body_len) { + u32 key_pos = json_str_value(body, body_len, k_jsonrpc_key, k_jsonrpc_key_len); + if (key_pos == INVALID_POS) + return 0; + + bpf_dbg_printk("Found JSON-RPC 2.0 key"); + + u32 val_search_start = json_value_offset(body, body_len, key_pos + k_jsonrpc_key_len); + // The jsonrpc value should be a string + if (val_search_start >= body_len || body[val_search_start] != '"') + return 0; + + u32 val_pos = json_str_value( + body + val_search_start, body_len - val_search_start, k_jsonrpc_val, k_jsonrpc_val_len); + // The jsonrpc value should start immediately after the opening quote + if (val_pos == INVALID_POS || val_pos != 0) + return 0; + + bpf_dbg_printk("Found JSON-RPC 2.0 value"); + + return 1; // JSON-RPC 2.0 detected +} + +// Extracts the value of the "method" key from a JSON-RPC 2.0 body. +// Returns the length of the method value, or 0 if not found or error. +// method_buf must be at least method_buf_len bytes. +static __always_inline u32 extract_jsonrpc2_method(const char *body, + u32 body_len, + char *method_buf) { + u32 key_pos = json_str_value(body, body_len, k_method_key, k_method_key_len); + if (key_pos == INVALID_POS) + return 0; + + bpf_dbg_printk("Found JSON-RPC method key"); + + u32 val_search_start = json_value_offset(body, body_len, key_pos + k_method_key_len); + // method value should be a string + if (val_search_start >= body_len || body[val_search_start] != '"') + return 0; + + bpf_dbg_printk("Found JSON-RPC method value opening quote"); + + // Copy the method value from the body after the opening quote + u32 value_start = val_search_start + 1; + u32 value_end = json_str_value_end(body, body_len, value_start); + + return copy_json_string_value( + body, value_start, value_end, method_buf, JSONRPC_METHOD_BUF_SIZE); +} \ No newline at end of file diff --git a/pkg/components/ebpf/gotracer/gotracer.go b/pkg/components/ebpf/gotracer/gotracer.go index 0f7d752874..2581ae00d7 100644 --- a/pkg/components/ebpf/gotracer/gotracer.go +++ b/pkg/components/ebpf/gotracer/gotracer.go @@ -205,6 +205,10 @@ func (p *Tracer) GoProbes() map[string][]*ebpfcommon.ProbeDesc { Start: p.bpfObjects.BeylaUprobeReadRequestStart, End: p.bpfObjects.BeylaUprobeReadRequestReturns, }}, + "net/http.(*body).Read": {{ + Start: p.bpfObjects.BeylaUprobeBodyRead, + End: p.bpfObjects.BeylaUprobeBodyReadReturn, + }}, "net/textproto.(*Reader).readContinuedLineSlice": {{ End: p.bpfObjects.BeylaUprobeReadContinuedLineSliceReturns, }}, From 77d6069b218a1e78ca0e5dc912b621aec7f79ac0 Mon Sep 17 00:00:00 2001 From: titaneric Date: Mon, 16 Jun 2025 17:18:59 +0000 Subject: [PATCH 02/40] fix clang lint error --- bpf/common/tc_common.h | 9 ++++++--- bpf/gotracer/go_nethttp.c | 3 ++- bpf/gotracer/protocol_jsonrpc.h | 26 +++++++++++++++++--------- 3 files changed, 25 insertions(+), 13 deletions(-) diff --git a/bpf/common/tc_common.h b/bpf/common/tc_common.h index 2e18bf75ef..8eda53b260 100644 --- a/bpf/common/tc_common.h +++ b/bpf/common/tc_common.h @@ -137,22 +137,25 @@ static __always_inline u32 bpf_memstr(const char *haystack, int haystack_len, const char *needle, int needle_len) { - if (needle_len == 0 || haystack_len < needle_len) + if (needle_len == 0 || haystack_len < needle_len) { return INVALID_POS; + } for (int i = 0; i <= haystack_len - needle_len; i++) { int found = 1; #pragma unroll // max needle length for (int j = 0; j < MAX_NEEDLE_LEN; j++) { - if (j >= needle_len) + if (j >= needle_len) { break; + } if (haystack[i + j] != needle[j]) { found = 0; break; } } - if (found) + if (found) { return (u32)i; + } } return INVALID_POS; } \ No newline at end of file diff --git a/bpf/gotracer/go_nethttp.c b/bpf/gotracer/go_nethttp.c index acd120a9b8..162408b391 100644 --- a/bpf/gotracer/go_nethttp.c +++ b/bpf/gotracer/go_nethttp.c @@ -295,8 +295,9 @@ int beyla_uprobe_readContinuedLineSliceReturns(struct pt_regs *ctx) { const unsigned char *buf = (const unsigned char *)GO_PARAM1(ctx); u8 temp[HTTP_HEADER_MAX_LEN]; - if (len > sizeof(temp)) + if (len > sizeof(temp)) { len = sizeof(temp); + } bpf_probe_read(temp, len, buf); bpf_dbg_printk("goroutine_addr %lx", goroutine_addr); diff --git a/bpf/gotracer/protocol_jsonrpc.h b/bpf/gotracer/protocol_jsonrpc.h index 13dd832100..f150b58bb5 100644 --- a/bpf/gotracer/protocol_jsonrpc.h +++ b/bpf/gotracer/protocol_jsonrpc.h @@ -49,7 +49,7 @@ static __always_inline u32 json_str_value(const char *body, u32 body_len, const char *str, u32 str_len) { - return bpf_memstr((const char *)body, body_len, (const char *)str, str_len); + return bpf_memstr(body, body_len, str, str_len); } // Returns the end position (index of closing quote) of a JSON string value. @@ -69,15 +69,18 @@ static __always_inline u32 json_str_value_end(const char *body, u32 body_len, u3 static __always_inline u32 copy_json_string_value( const char *body, u32 value_start, u32 value_end, char *dest_buf, u32 dest_buf_size) { u32 value_len = value_end - value_start; - if (value_len <= 0) + if (value_len <= 0) { return 0; - if (value_len >= dest_buf_size) + } + if (value_len >= dest_buf_size) { value_len = dest_buf_size - 1; // leave space for null terminator + } #pragma unroll for (u32 i = 0; i < dest_buf_size; i++) { - if (i >= value_len) + if (i >= value_len) { break; + } dest_buf[i] = body[value_start + i]; } dest_buf[value_len] = '\0'; @@ -87,21 +90,24 @@ static __always_inline u32 copy_json_string_value( // Looks for '"jsonrpc":"2.0"' static __always_inline u32 is_jsonrpc2_body(const char *body, u32 body_len) { u32 key_pos = json_str_value(body, body_len, k_jsonrpc_key, k_jsonrpc_key_len); - if (key_pos == INVALID_POS) + if (key_pos == INVALID_POS) { return 0; + } bpf_dbg_printk("Found JSON-RPC 2.0 key"); u32 val_search_start = json_value_offset(body, body_len, key_pos + k_jsonrpc_key_len); // The jsonrpc value should be a string - if (val_search_start >= body_len || body[val_search_start] != '"') + if (val_search_start >= body_len || body[val_search_start] != '"') { return 0; + } u32 val_pos = json_str_value( body + val_search_start, body_len - val_search_start, k_jsonrpc_val, k_jsonrpc_val_len); // The jsonrpc value should start immediately after the opening quote - if (val_pos == INVALID_POS || val_pos != 0) + if (val_pos == INVALID_POS || val_pos != 0) { return 0; + } bpf_dbg_printk("Found JSON-RPC 2.0 value"); @@ -115,15 +121,17 @@ static __always_inline u32 extract_jsonrpc2_method(const char *body, u32 body_len, char *method_buf) { u32 key_pos = json_str_value(body, body_len, k_method_key, k_method_key_len); - if (key_pos == INVALID_POS) + if (key_pos == INVALID_POS) { return 0; + } bpf_dbg_printk("Found JSON-RPC method key"); u32 val_search_start = json_value_offset(body, body_len, key_pos + k_method_key_len); // method value should be a string - if (val_search_start >= body_len || body[val_search_start] != '"') + if (val_search_start >= body_len || body[val_search_start] != '"') { return 0; + } bpf_dbg_printk("Found JSON-RPC method value opening quote"); From d371169f1350e3736812d67d78782fd8c8e0a843 Mon Sep 17 00:00:00 2001 From: titaneric Date: Tue, 17 Jun 2025 15:55:41 +0000 Subject: [PATCH 03/40] avoid variable length buffer read in readContinuedLineSlice --- bpf/gotracer/go_nethttp.c | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/bpf/gotracer/go_nethttp.c b/bpf/gotracer/go_nethttp.c index 162408b391..b3dc54d0c4 100644 --- a/bpf/gotracer/go_nethttp.c +++ b/bpf/gotracer/go_nethttp.c @@ -294,11 +294,16 @@ int beyla_uprobe_readContinuedLineSliceReturns(struct pt_regs *ctx) { u64 len = (u64)GO_PARAM2(ctx); const unsigned char *buf = (const unsigned char *)GO_PARAM1(ctx); - u8 temp[HTTP_HEADER_MAX_LEN]; - if (len > sizeof(temp)) { - len = sizeof(temp); + // avoid variable-length buffer reads to work on older kernels such as 5.15 or earlier + u8 temp[HTTP_HEADER_MAX_LEN] = {}; + u64 safe_len = len > HTTP_HEADER_MAX_LEN ? HTTP_HEADER_MAX_LEN : len; + +#pragma unroll + for (int i = 0; i < HTTP_HEADER_MAX_LEN; i++) { + if (i < safe_len) { + bpf_probe_read(&temp[i], 1, buf + i); + } } - bpf_probe_read(temp, len, buf); bpf_dbg_printk("goroutine_addr %lx", goroutine_addr); go_addr_key_t g_key = {}; @@ -315,7 +320,7 @@ int beyla_uprobe_readContinuedLineSliceReturns(struct pt_regs *ctx) { } server_http_func_invocation_t *inv = bpf_map_lookup_elem(&ongoing_http_server_requests, &g_key); - if (len >= w3c_header_length && + if (safe_len >= w3c_header_length && !bpf_memicmp((const char *)temp, "traceparent: ", w3c_value_start)) { u8 *traceparent_start = temp + w3c_value_start; if (inv) { @@ -325,7 +330,7 @@ int beyla_uprobe_readContinuedLineSliceReturns(struct pt_regs *ctx) { update_traceparent(&minimal_inv, traceparent_start); bpf_map_update_elem(&ongoing_http_server_requests, &g_key, &minimal_inv, BPF_ANY); } - } else if (len >= content_type_header_length && + } else if (safe_len >= content_type_header_length && !bpf_memicmp((const char *)temp, "content-type: ", content_type_value_start)) { u8 *content_type_start = temp + content_type_value_start; if (inv) { From dd7d476691e5046ed14a24317807f04d20f7ede7 Mon Sep 17 00:00:00 2001 From: titaneric Date: Wed, 18 Jun 2025 11:56:13 +0000 Subject: [PATCH 04/40] convert const char to const unsigned char --- bpf/common/tc_common.h | 8 ++++---- bpf/gotracer/go_nethttp.c | 6 +++--- bpf/gotracer/protocol_jsonrpc.h | 30 +++++++++++++++++++----------- 3 files changed, 26 insertions(+), 18 deletions(-) diff --git a/bpf/common/tc_common.h b/bpf/common/tc_common.h index 8eda53b260..3d88f4bf05 100644 --- a/bpf/common/tc_common.h +++ b/bpf/common/tc_common.h @@ -133,10 +133,10 @@ static __always_inline u32 sk_msg_local_port(struct sk_msg_md *ctx) { } // find the needle in the haystack, return the position of the first occurrence, return -1 if not found -static __always_inline u32 bpf_memstr(const char *haystack, - int haystack_len, - const char *needle, - int needle_len) { +static __always_inline u32 bpf_memstr(const unsigned char *haystack, + u32 haystack_len, + const unsigned char *needle, + u32 needle_len) { if (needle_len == 0 || haystack_len < needle_len) { return INVALID_POS; } diff --git a/bpf/gotracer/go_nethttp.c b/bpf/gotracer/go_nethttp.c index b3dc54d0c4..a5cf97378d 100644 --- a/bpf/gotracer/go_nethttp.c +++ b/bpf/gotracer/go_nethttp.c @@ -1281,10 +1281,10 @@ int beyla_uprobe_bodyReadReturn(struct pt_regs *ctx) { if (read_go_str_n( "http body", (void *)invocation->body_addr, n, body_buf, sizeof(body_buf))) { bpf_dbg_printk("body is %s", body_buf); - if (is_jsonrpc2_body(body_buf, sizeof(body_buf))) { + if (is_jsonrpc2_body((const unsigned char *)body_buf, sizeof(body_buf))) { char method_buf[JSONRPC_METHOD_BUF_SIZE] = {}; - u32 method_len = - extract_jsonrpc2_method(body_buf, sizeof(body_buf), method_buf); + u32 method_len = extract_jsonrpc2_method( + (const unsigned char *)body_buf, sizeof(body_buf), method_buf); if (method_len > 0) { bpf_dbg_printk("JSON-RPC method: %s", method_buf); read_go_str_n("JSON-RPC method", diff --git a/bpf/gotracer/protocol_jsonrpc.h b/bpf/gotracer/protocol_jsonrpc.h index f150b58bb5..afca90adfd 100644 --- a/bpf/gotracer/protocol_jsonrpc.h +++ b/bpf/gotracer/protocol_jsonrpc.h @@ -35,7 +35,9 @@ static __always_inline u8 bpf_isspace(char c) { // Returns the offset of the next JSON value after skipping whitespace and colon. // If not found, returns body_len. -static __always_inline u32 json_value_offset(const char *body, u32 body_len, u32 start_pos) { +static __always_inline u32 json_value_offset(const unsigned char *body, + u32 body_len, + u32 start_pos) { u32 pos = start_pos; while (pos < body_len && (bpf_isspace(body[pos]) || body[pos] == ':')) { pos++; @@ -45,16 +47,18 @@ static __always_inline u32 json_value_offset(const char *body, u32 body_len, u32 // Returns the position of the first occurrence of a string in a JSON body. // If not found, returns INVALID_POS. -static __always_inline u32 json_str_value(const char *body, +static __always_inline u32 json_str_value(const unsigned char *body, u32 body_len, - const char *str, + const unsigned char *str, u32 str_len) { return bpf_memstr(body, body_len, str, str_len); } // Returns the end position (index of closing quote) of a JSON string value. // If not found, returns body_len. -static __always_inline u32 json_str_value_end(const char *body, u32 body_len, u32 value_start) { +static __always_inline u32 json_str_value_end(const unsigned char *body, + u32 body_len, + u32 value_start) { // find_first_pos_of expects unsigned char*, so cast accordingly return value_start + find_first_pos_of((unsigned char *)(body + value_start), (unsigned char *)(body + body_len), @@ -67,7 +71,7 @@ static __always_inline u32 json_str_value_end(const char *body, u32 body_len, u3 * Returns the number of bytes copied (excluding null terminator), or 0 on error. */ static __always_inline u32 copy_json_string_value( - const char *body, u32 value_start, u32 value_end, char *dest_buf, u32 dest_buf_size) { + const unsigned char *body, u32 value_start, u32 value_end, char *dest_buf, u32 dest_buf_size) { u32 value_len = value_end - value_start; if (value_len <= 0) { return 0; @@ -88,8 +92,9 @@ static __always_inline u32 copy_json_string_value( } // Looks for '"jsonrpc":"2.0"' -static __always_inline u32 is_jsonrpc2_body(const char *body, u32 body_len) { - u32 key_pos = json_str_value(body, body_len, k_jsonrpc_key, k_jsonrpc_key_len); +static __always_inline u32 is_jsonrpc2_body(const unsigned char *body, u32 body_len) { + u32 key_pos = + json_str_value(body, body_len, (const unsigned char *)k_jsonrpc_key, k_jsonrpc_key_len); if (key_pos == INVALID_POS) { return 0; } @@ -102,8 +107,10 @@ static __always_inline u32 is_jsonrpc2_body(const char *body, u32 body_len) { return 0; } - u32 val_pos = json_str_value( - body + val_search_start, body_len - val_search_start, k_jsonrpc_val, k_jsonrpc_val_len); + u32 val_pos = json_str_value(body + val_search_start, + body_len - val_search_start, + (const unsigned char *)k_jsonrpc_val, + k_jsonrpc_val_len); // The jsonrpc value should start immediately after the opening quote if (val_pos == INVALID_POS || val_pos != 0) { return 0; @@ -117,10 +124,11 @@ static __always_inline u32 is_jsonrpc2_body(const char *body, u32 body_len) { // Extracts the value of the "method" key from a JSON-RPC 2.0 body. // Returns the length of the method value, or 0 if not found or error. // method_buf must be at least method_buf_len bytes. -static __always_inline u32 extract_jsonrpc2_method(const char *body, +static __always_inline u32 extract_jsonrpc2_method(const unsigned char *body, u32 body_len, char *method_buf) { - u32 key_pos = json_str_value(body, body_len, k_method_key, k_method_key_len); + u32 key_pos = + json_str_value(body, body_len, (const unsigned char *)k_method_key, k_method_key_len); if (key_pos == INVALID_POS) { return 0; } From 1f762dcc42ecde25bc3d202690109d205b14cf44 Mon Sep 17 00:00:00 2001 From: titaneric Date: Wed, 18 Jun 2025 13:42:45 +0000 Subject: [PATCH 05/40] slightly type change in bpf_memstr --- bpf/common/tc_common.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bpf/common/tc_common.h b/bpf/common/tc_common.h index 3d88f4bf05..bad51a47af 100644 --- a/bpf/common/tc_common.h +++ b/bpf/common/tc_common.h @@ -140,11 +140,11 @@ static __always_inline u32 bpf_memstr(const unsigned char *haystack, if (needle_len == 0 || haystack_len < needle_len) { return INVALID_POS; } - for (int i = 0; i <= haystack_len - needle_len; i++) { - int found = 1; + for (u32 i = 0; i <= haystack_len - needle_len; i++) { + u8 found = 1; #pragma unroll // max needle length - for (int j = 0; j < MAX_NEEDLE_LEN; j++) { + for (u8 j = 0; j < MAX_NEEDLE_LEN; j++) { if (j >= needle_len) { break; } @@ -154,7 +154,7 @@ static __always_inline u32 bpf_memstr(const unsigned char *haystack, } } if (found) { - return (u32)i; + return i; } } return INVALID_POS; From b90326d9f4f8dbd50b525271205c61f00e19b830 Mon Sep 17 00:00:00 2001 From: titaneric Date: Wed, 18 Jun 2025 13:55:41 +0000 Subject: [PATCH 06/40] make both update traceparent and content type to const unsigned char --- bpf/gotracer/go_common.h | 8 ++++---- bpf/gotracer/go_nethttp.c | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/bpf/gotracer/go_common.h b/bpf/gotracer/go_common.h index ac7b28cbe8..76e4cb0c3c 100644 --- a/bpf/gotracer/go_common.h +++ b/bpf/gotracer/go_common.h @@ -192,14 +192,14 @@ static __always_inline u64 find_parent_goroutine_in_chain(go_addr_key_t *current return 0; } -static __always_inline void decode_go_traceparent(unsigned char *buf, +static __always_inline void decode_go_traceparent(const unsigned char *buf, unsigned char *trace_id, unsigned char *span_id, unsigned char *flags) { - unsigned char *t_id = buf + 2 + 1; // strlen(ver) + strlen("-") - unsigned char *s_id = + const unsigned char *t_id = buf + 2 + 1; // strlen(ver) + strlen("-") + const unsigned char *s_id = buf + 2 + 1 + 32 + 1; // strlen(ver) + strlen("-") + strlen(trace_id) + strlen("-") - unsigned char *f_id = + const unsigned char *f_id = buf + 2 + 1 + 32 + 1 + 16 + 1; // strlen(ver) + strlen("-") + strlen(trace_id) + strlen("-") + strlen(span_id) + strlen("-") diff --git a/bpf/gotracer/go_nethttp.c b/bpf/gotracer/go_nethttp.c index a5cf97378d..876b32c82c 100644 --- a/bpf/gotracer/go_nethttp.c +++ b/bpf/gotracer/go_nethttp.c @@ -274,13 +274,13 @@ int beyla_uprobe_http2Server_processHeaders(struct pt_regs *ctx) { } static __always_inline void update_traceparent(server_http_func_invocation_t *inv, - u8 *header_start) { + const unsigned char *header_start) { decode_go_traceparent(header_start, inv->tp.trace_id, inv->tp.parent_id, &inv->tp.flags); bpf_dbg_printk("Found traceparent in header %s", header_start); } static __always_inline void update_content_type(server_http_func_invocation_t *inv, - u8 *header_start) { + const unsigned char *header_start) { __builtin_memset(inv->content_type, 0, sizeof(inv->content_type)); __builtin_memcpy(inv->content_type, header_start, sizeof(inv->content_type)); bpf_dbg_printk("Found content-type in header %s", inv->content_type); From 5849e6605c66159dc82b2558332c2d5e8658d575 Mon Sep 17 00:00:00 2001 From: titaneric Date: Fri, 20 Jun 2025 14:32:25 +0000 Subject: [PATCH 07/40] add temp bpf map and refactor buffer read in readContinuedLineSlice --- bpf/gotracer/go_nethttp.c | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/bpf/gotracer/go_nethttp.c b/bpf/gotracer/go_nethttp.c index 876b32c82c..c139160746 100644 --- a/bpf/gotracer/go_nethttp.c +++ b/bpf/gotracer/go_nethttp.c @@ -79,6 +79,18 @@ struct { __uint(max_entries, MAX_CONCURRENT_REQUESTS); } ongoing_http_server_requests SEC(".maps"); +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); + __type(key, u32); + __type(value, unsigned char[HTTP_HEADER_MAX_LEN]); + __uint(max_entries, 1); +} temp_mem_scratch SEC(".maps"); + +static __always_inline unsigned char *temp_mem() { + const u32 zero = 0; + return bpf_map_lookup_elem(&temp_mem_scratch, &zero); +} + /* HTTP Server */ // This instrumentation attaches uprobe to the following function: @@ -294,16 +306,12 @@ int beyla_uprobe_readContinuedLineSliceReturns(struct pt_regs *ctx) { u64 len = (u64)GO_PARAM2(ctx); const unsigned char *buf = (const unsigned char *)GO_PARAM1(ctx); - // avoid variable-length buffer reads to work on older kernels such as 5.15 or earlier - u8 temp[HTTP_HEADER_MAX_LEN] = {}; - u64 safe_len = len > HTTP_HEADER_MAX_LEN ? HTTP_HEADER_MAX_LEN : len; - -#pragma unroll - for (int i = 0; i < HTTP_HEADER_MAX_LEN; i++) { - if (i < safe_len) { - bpf_probe_read(&temp[i], 1, buf + i); - } - } + unsigned char *temp = temp_mem(); + const u32 safe_len = len > HTTP_HEADER_MAX_LEN ? HTTP_HEADER_MAX_LEN : len; + if (!temp || bpf_probe_read_user(temp, safe_len, buf) != 0) { + bpf_dbg_printk("failed to read buffer"); + return 0; + }; bpf_dbg_printk("goroutine_addr %lx", goroutine_addr); go_addr_key_t g_key = {}; From e17ccc8736b8b81e82c4cfcac0b7f21448cea64f Mon Sep 17 00:00:00 2001 From: titaneric Date: Fri, 20 Jun 2025 14:56:10 +0000 Subject: [PATCH 08/40] break early if ongoing_server_connections not found --- bpf/gotracer/go_nethttp.c | 62 +++++++++++++++++++-------------------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/bpf/gotracer/go_nethttp.c b/bpf/gotracer/go_nethttp.c index c139160746..757966470b 100644 --- a/bpf/gotracer/go_nethttp.c +++ b/bpf/gotracer/go_nethttp.c @@ -303,6 +303,14 @@ int beyla_uprobe_readContinuedLineSliceReturns(struct pt_regs *ctx) { bpf_dbg_printk("=== uprobe/proc readContinuedLineSlice returns === "); void *goroutine_addr = GOROUTINE_PTR(ctx); + bpf_dbg_printk("goroutine_addr %lx", goroutine_addr); + go_addr_key_t g_key = {}; + go_addr_key_from_id(&g_key, goroutine_addr); + connection_info_t *existing = bpf_map_lookup_elem(&ongoing_server_connections, &g_key); + if (!existing) { + return 0; + } + u64 len = (u64)GO_PARAM2(ctx); const unsigned char *buf = (const unsigned char *)GO_PARAM1(ctx); @@ -313,19 +321,11 @@ int beyla_uprobe_readContinuedLineSliceReturns(struct pt_regs *ctx) { return 0; }; - bpf_dbg_printk("goroutine_addr %lx", goroutine_addr); - go_addr_key_t g_key = {}; - go_addr_key_from_id(&g_key, goroutine_addr); - - int w3c_value_start = W3C_KEY_LENGTH + 2; // "traceparent: " - int w3c_header_length = w3c_value_start + W3C_VAL_LENGTH; - int content_type_value_start = CONTENT_TYPE_KEY_LEN + 2; // "content-type: " - int content_type_header_length = content_type_value_start + HTTP_CONTENT_TYPE_MAX_LEN; + const u32 w3c_value_start = W3C_KEY_LENGTH + 2; // "traceparent: " + const u32 w3c_header_length = w3c_value_start + W3C_VAL_LENGTH; + const u32 content_type_value_start = CONTENT_TYPE_KEY_LEN + 2; // "content-type: " + const u32 content_type_header_length = content_type_value_start + HTTP_CONTENT_TYPE_MAX_LEN; - connection_info_t *existing = bpf_map_lookup_elem(&ongoing_server_connections, &g_key); - if (!existing) { - return 0; - } server_http_func_invocation_t *inv = bpf_map_lookup_elem(&ongoing_http_server_requests, &g_key); if (safe_len >= w3c_header_length && @@ -1279,28 +1279,28 @@ int beyla_uprobe_bodyReadReturn(struct pt_regs *ctx) { return 0; } // content-type is set in invocation in ServeHTTP - bpf_dbg_printk("n is %d", n); + bpf_dbg_printk("n is %llu", n); bpf_dbg_printk("content type is %s", invocation->content_type); char body_buf[HTTP_BODY_MAX_LEN] = {}; - if (n > 0 && invocation->body_addr) { - if (is_json_content_type((void *)invocation->content_type, - sizeof(invocation->content_type))) { - if (read_go_str_n( - "http body", (void *)invocation->body_addr, n, body_buf, sizeof(body_buf))) { - bpf_dbg_printk("body is %s", body_buf); - if (is_jsonrpc2_body((const unsigned char *)body_buf, sizeof(body_buf))) { - char method_buf[JSONRPC_METHOD_BUF_SIZE] = {}; - u32 method_len = extract_jsonrpc2_method( - (const unsigned char *)body_buf, sizeof(body_buf), method_buf); - if (method_len > 0) { - bpf_dbg_printk("JSON-RPC method: %s", method_buf); - read_go_str_n("JSON-RPC method", - (void *)method_buf, - method_len, - invocation->method, - sizeof(invocation->method)); - } + if (n <= 0 || !invocation->body_addr) { + return 0; + } + if (is_json_content_type((void *)invocation->content_type, sizeof(invocation->content_type))) { + if (read_go_str_n( + "http body", (void *)invocation->body_addr, n, body_buf, sizeof(body_buf))) { + bpf_dbg_printk("body is %s", body_buf); + if (is_jsonrpc2_body((const unsigned char *)body_buf, sizeof(body_buf))) { + char method_buf[JSONRPC_METHOD_BUF_SIZE] = {}; + u32 method_len = extract_jsonrpc2_method( + (const unsigned char *)body_buf, sizeof(body_buf), method_buf); + if (method_len > 0) { + bpf_dbg_printk("JSON-RPC method: %s", method_buf); + read_go_str_n("JSON-RPC method", + (void *)method_buf, + method_len, + invocation->method, + sizeof(invocation->method)); } } } From 523369303a5d5a49cae0b6302d58061b13c9d2bb Mon Sep 17 00:00:00 2001 From: titaneric Date: Fri, 20 Jun 2025 15:37:53 +0000 Subject: [PATCH 09/40] calculate http header by sizeof --- bpf/gotracer/go_nethttp.c | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/bpf/gotracer/go_nethttp.c b/bpf/gotracer/go_nethttp.c index 757966470b..d025094a85 100644 --- a/bpf/gotracer/go_nethttp.c +++ b/bpf/gotracer/go_nethttp.c @@ -321,16 +321,19 @@ int beyla_uprobe_readContinuedLineSliceReturns(struct pt_regs *ctx) { return 0; }; - const u32 w3c_value_start = W3C_KEY_LENGTH + 2; // "traceparent: " + const char traceparent[] = "traceparent: "; + const char content_type[] = "content-type: "; + + const u32 w3c_value_start = sizeof(traceparent) - 1; const u32 w3c_header_length = w3c_value_start + W3C_VAL_LENGTH; - const u32 content_type_value_start = CONTENT_TYPE_KEY_LEN + 2; // "content-type: " + const u32 content_type_value_start = sizeof(content_type) - 1; const u32 content_type_header_length = content_type_value_start + HTTP_CONTENT_TYPE_MAX_LEN; server_http_func_invocation_t *inv = bpf_map_lookup_elem(&ongoing_http_server_requests, &g_key); if (safe_len >= w3c_header_length && - !bpf_memicmp((const char *)temp, "traceparent: ", w3c_value_start)) { - u8 *traceparent_start = temp + w3c_value_start; + bpf_memicmp((const char *)temp, (const char *)traceparent, w3c_value_start) == 0) { + unsigned char *traceparent_start = temp + w3c_value_start; if (inv) { update_traceparent(inv, traceparent_start); } else { @@ -339,8 +342,9 @@ int beyla_uprobe_readContinuedLineSliceReturns(struct pt_regs *ctx) { bpf_map_update_elem(&ongoing_http_server_requests, &g_key, &minimal_inv, BPF_ANY); } } else if (safe_len >= content_type_header_length && - !bpf_memicmp((const char *)temp, "content-type: ", content_type_value_start)) { - u8 *content_type_start = temp + content_type_value_start; + bpf_memicmp( + (const char *)temp, (const char *)content_type, content_type_value_start) == 0) { + unsigned char *content_type_start = temp + content_type_value_start; if (inv) { update_content_type(inv, content_type_start); } else { From 93ed4fb5d36a9f168a82647ae7ed534e4a3db8ef Mon Sep 17 00:00:00 2001 From: titaneric Date: Fri, 20 Jun 2025 15:58:10 +0000 Subject: [PATCH 10/40] refactor readContinuedLineSlice --- bpf/gotracer/go_nethttp.c | 67 +++++++++++++++++++++++++-------------- 1 file changed, 44 insertions(+), 23 deletions(-) diff --git a/bpf/gotracer/go_nethttp.c b/bpf/gotracer/go_nethttp.c index d025094a85..399d2bd45e 100644 --- a/bpf/gotracer/go_nethttp.c +++ b/bpf/gotracer/go_nethttp.c @@ -298,6 +298,40 @@ static __always_inline void update_content_type(server_http_func_invocation_t *i bpf_dbg_printk("Found content-type in header %s", inv->content_type); } +static __always_inline void handle_traceparent_header(server_http_func_invocation_t *inv, + go_addr_key_t *g_key, + unsigned char *traceparent_start) { + if (inv) { + update_traceparent(inv, traceparent_start); + } else { + server_http_func_invocation_t minimal_inv = {}; + update_traceparent(&minimal_inv, traceparent_start); + bpf_map_update_elem(&ongoing_http_server_requests, g_key, &minimal_inv, BPF_ANY); + } +} + +static __always_inline void handle_content_type_header(server_http_func_invocation_t *inv, + go_addr_key_t *g_key, + unsigned char *content_type_start) { + if (inv) { + update_content_type(inv, content_type_start); + } else { + server_http_func_invocation_t minimal_inv = {}; + update_content_type(&minimal_inv, content_type_start); + bpf_map_update_elem(&ongoing_http_server_requests, g_key, &minimal_inv, BPF_ANY); + } +} + +// Matches the header in the buffer and returns a pointer to the value part of the header. +static __always_inline unsigned char *match_header( + const unsigned char *buf, u32 safe_len, const char *header, u32 header_len, u32 value_len) { + if (safe_len >= header_len + value_len && + bpf_memicmp((const char *)buf, header, header_len) == 0) { + return (unsigned char *)(buf + header_len); + } + return NULL; +} + SEC("uprobe/readContinuedLineSlice") int beyla_uprobe_readContinuedLineSliceReturns(struct pt_regs *ctx) { bpf_dbg_printk("=== uprobe/proc readContinuedLineSlice returns === "); @@ -325,33 +359,20 @@ int beyla_uprobe_readContinuedLineSliceReturns(struct pt_regs *ctx) { const char content_type[] = "content-type: "; const u32 w3c_value_start = sizeof(traceparent) - 1; - const u32 w3c_header_length = w3c_value_start + W3C_VAL_LENGTH; const u32 content_type_value_start = sizeof(content_type) - 1; - const u32 content_type_header_length = content_type_value_start + HTTP_CONTENT_TYPE_MAX_LEN; server_http_func_invocation_t *inv = bpf_map_lookup_elem(&ongoing_http_server_requests, &g_key); - if (safe_len >= w3c_header_length && - bpf_memicmp((const char *)temp, (const char *)traceparent, w3c_value_start) == 0) { - unsigned char *traceparent_start = temp + w3c_value_start; - if (inv) { - update_traceparent(inv, traceparent_start); - } else { - server_http_func_invocation_t minimal_inv = {}; - update_traceparent(&minimal_inv, traceparent_start); - bpf_map_update_elem(&ongoing_http_server_requests, &g_key, &minimal_inv, BPF_ANY); - } - } else if (safe_len >= content_type_header_length && - bpf_memicmp( - (const char *)temp, (const char *)content_type, content_type_value_start) == 0) { - unsigned char *content_type_start = temp + content_type_value_start; - if (inv) { - update_content_type(inv, content_type_start); - } else { - server_http_func_invocation_t minimal_inv = {}; - update_content_type(&minimal_inv, content_type_start); - bpf_map_update_elem(&ongoing_http_server_requests, &g_key, &minimal_inv, BPF_ANY); - } + unsigned char *traceparent_start = + match_header(temp, safe_len, traceparent, w3c_value_start, W3C_VAL_LENGTH); + if (traceparent_start) { + handle_traceparent_header(inv, &g_key, traceparent_start); + } + + unsigned char *content_type_start = match_header( + temp, safe_len, content_type, content_type_value_start, HTTP_CONTENT_TYPE_MAX_LEN); + if (content_type_start) { + handle_content_type_header(inv, &g_key, content_type_start); } return 0; From 9a7cb1b86340e8cde54452c3734e343a73098673 Mon Sep 17 00:00:00 2001 From: titaneric Date: Fri, 20 Jun 2025 16:29:59 +0000 Subject: [PATCH 11/40] rename temp_mem map --- bpf/gotracer/go_nethttp.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bpf/gotracer/go_nethttp.c b/bpf/gotracer/go_nethttp.c index 399d2bd45e..afa92a1555 100644 --- a/bpf/gotracer/go_nethttp.c +++ b/bpf/gotracer/go_nethttp.c @@ -84,11 +84,11 @@ struct { __type(key, u32); __type(value, unsigned char[HTTP_HEADER_MAX_LEN]); __uint(max_entries, 1); -} temp_mem_scratch SEC(".maps"); +} temp_header_mem_store SEC(".maps"); -static __always_inline unsigned char *temp_mem() { +static __always_inline unsigned char *temp_header_mem() { const u32 zero = 0; - return bpf_map_lookup_elem(&temp_mem_scratch, &zero); + return bpf_map_lookup_elem(&temp_header_mem_store, &zero); } /* HTTP Server */ @@ -348,7 +348,7 @@ int beyla_uprobe_readContinuedLineSliceReturns(struct pt_regs *ctx) { u64 len = (u64)GO_PARAM2(ctx); const unsigned char *buf = (const unsigned char *)GO_PARAM1(ctx); - unsigned char *temp = temp_mem(); + unsigned char *temp = temp_header_mem(); const u32 safe_len = len > HTTP_HEADER_MAX_LEN ? HTTP_HEADER_MAX_LEN : len; if (!temp || bpf_probe_read_user(temp, safe_len, buf) != 0) { bpf_dbg_printk("failed to read buffer"); From 0461ae2ec85f033c2cc5e77ca5f648a20d9b50d1 Mon Sep 17 00:00:00 2001 From: titaneric Date: Thu, 19 Jun 2025 18:24:54 +0000 Subject: [PATCH 12/40] attempt jsonrpc tail code and introduce body bpg map --- bpf/gotracer/go_nethttp.c | 86 ++++++++++++++++++------ bpf/gotracer/maps/jsonrpc_jump_table.h | 11 +++ bpf/gotracer/protocol_jsonrpc.h | 9 ++- pkg/components/ebpf/gotracer/gotracer.go | 17 ++++- 4 files changed, 100 insertions(+), 23 deletions(-) create mode 100644 bpf/gotracer/maps/jsonrpc_jump_table.h diff --git a/bpf/gotracer/go_nethttp.c b/bpf/gotracer/go_nethttp.c index afa92a1555..66cf0078ec 100644 --- a/bpf/gotracer/go_nethttp.c +++ b/bpf/gotracer/go_nethttp.c @@ -27,6 +27,7 @@ #include #include #include +#include #include @@ -35,6 +36,8 @@ #include +enum { k_tail_jsonrpc = 0 }; + typedef struct http_client_data { s64 content_length; pid_info pid; @@ -90,6 +93,17 @@ static __always_inline unsigned char *temp_header_mem() { const u32 zero = 0; return bpf_map_lookup_elem(&temp_header_mem_store, &zero); } +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); + __type(key, u32); + __type(value, unsigned char[HTTP_BODY_MAX_LEN]); + __uint(max_entries, 1); +} temp_body_mem_store SEC(".maps"); + +static __always_inline unsigned char *temp_body_mem() { + const u32 zero = 0; + return bpf_map_lookup_elem(&temp_body_mem_store, &zero); +} /* HTTP Server */ @@ -1290,8 +1304,8 @@ int beyla_uprobe_bodyRead(struct pt_regs *ctx) { SEC("uprobe/bodyReadRet") int beyla_uprobe_bodyReadReturn(struct pt_regs *ctx) { - void *goroutine_addr = GOROUTINE_PTR(ctx); bpf_dbg_printk("=== uprobe/proc body read returns goroutine === "); + void *goroutine_addr = GOROUTINE_PTR(ctx); go_addr_key_t g_key = {}; go_addr_key_from_id(&g_key, goroutine_addr); @@ -1307,27 +1321,61 @@ int beyla_uprobe_bodyReadReturn(struct pt_regs *ctx) { bpf_dbg_printk("n is %llu", n); bpf_dbg_printk("content type is %s", invocation->content_type); - char body_buf[HTTP_BODY_MAX_LEN] = {}; if (n <= 0 || !invocation->body_addr) { return 0; } - if (is_json_content_type((void *)invocation->content_type, sizeof(invocation->content_type))) { - if (read_go_str_n( - "http body", (void *)invocation->body_addr, n, body_buf, sizeof(body_buf))) { - bpf_dbg_printk("body is %s", body_buf); - if (is_jsonrpc2_body((const unsigned char *)body_buf, sizeof(body_buf))) { - char method_buf[JSONRPC_METHOD_BUF_SIZE] = {}; - u32 method_len = extract_jsonrpc2_method( - (const unsigned char *)body_buf, sizeof(body_buf), method_buf); - if (method_len > 0) { - bpf_dbg_printk("JSON-RPC method: %s", method_buf); - read_go_str_n("JSON-RPC method", - (void *)method_buf, - method_len, - invocation->method, - sizeof(invocation->method)); - } - } + unsigned char *body_buf = temp_body_mem(); + if (!body_buf) { + return 0; + } + if (!is_json_content_type((void *)invocation->content_type, sizeof(invocation->content_type))) { + bpf_dbg_printk("content type is not json, skipping"); + return 0; + } + + const u32 safe_len = n > HTTP_BODY_MAX_LEN ? HTTP_BODY_MAX_LEN : n; + if (!read_go_str_n("http body", (void *)invocation->body_addr, n, body_buf, safe_len)) { + bpf_dbg_printk("failed to read body, n=%llu, body_addr=%llx", n, invocation->body_addr); + return 0; + } + bpf_dbg_printk("body is %s", body_buf); + bpf_tail_call(ctx, &jsonrpc_jump_table, k_tail_jsonrpc); + return 0; +} + +//k_tail_jsonrpc +SEC("uprobe/readJsonrpcMethod") +int beyla_read_jsonrpc_method(struct pt_regs *ctx) { + bpf_dbg_printk("=== uprobe/proc read jsonrpc method === "); + void *goroutine_addr = GOROUTINE_PTR(ctx); + go_addr_key_t g_key = {}; + go_addr_key_from_id(&g_key, goroutine_addr); + + server_http_func_invocation_t *invocation = + bpf_map_lookup_elem(&ongoing_http_server_requests, &g_key); + if (!invocation) { + bpf_dbg_printk("can't find invocation info for server call"); + return 0; + } + + unsigned char *body_map_buf = temp_body_mem(); + if (!body_map_buf) { + return 0; + } + // TODO: save this body buffer? + unsigned char body_buf[HTTP_BODY_MAX_LEN] = {}; + __builtin_memcpy(body_buf, body_map_buf, sizeof(body_buf)); + if (is_jsonrpc2_body((const unsigned char *)body_buf, sizeof(body_buf))) { + unsigned char method_buf[JSONRPC_METHOD_BUF_SIZE] = {}; + u32 method_len = + extract_jsonrpc2_method((const unsigned char *)body_buf, sizeof(body_buf), method_buf); + if (method_len > 0) { + bpf_dbg_printk("JSON-RPC method: %s", method_buf); + read_go_str_n("JSON-RPC method", + (void *)method_buf, + method_len, + invocation->method, + sizeof(invocation->method)); } } return 0; diff --git a/bpf/gotracer/maps/jsonrpc_jump_table.h b/bpf/gotracer/maps/jsonrpc_jump_table.h new file mode 100644 index 0000000000..02b8e88f75 --- /dev/null +++ b/bpf/gotracer/maps/jsonrpc_jump_table.h @@ -0,0 +1,11 @@ +#pragma once + +#include +#include + +struct { + __uint(type, BPF_MAP_TYPE_PROG_ARRAY); + __type(key, u32); + __type(value, u32); + __uint(max_entries, 1); +} jsonrpc_jump_table SEC(".maps"); diff --git a/bpf/gotracer/protocol_jsonrpc.h b/bpf/gotracer/protocol_jsonrpc.h index afca90adfd..c236172c97 100644 --- a/bpf/gotracer/protocol_jsonrpc.h +++ b/bpf/gotracer/protocol_jsonrpc.h @@ -70,8 +70,11 @@ static __always_inline u32 json_str_value_end(const unsigned char *body, * Ensures null-termination and does not exceed dest_buf_size. * Returns the number of bytes copied (excluding null terminator), or 0 on error. */ -static __always_inline u32 copy_json_string_value( - const unsigned char *body, u32 value_start, u32 value_end, char *dest_buf, u32 dest_buf_size) { +static __always_inline u32 copy_json_string_value(const unsigned char *body, + u32 value_start, + u32 value_end, + unsigned char *dest_buf, + u32 dest_buf_size) { u32 value_len = value_end - value_start; if (value_len <= 0) { return 0; @@ -126,7 +129,7 @@ static __always_inline u32 is_jsonrpc2_body(const unsigned char *body, u32 body_ // method_buf must be at least method_buf_len bytes. static __always_inline u32 extract_jsonrpc2_method(const unsigned char *body, u32 body_len, - char *method_buf) { + unsigned char *method_buf) { u32 key_pos = json_str_value(body, body_len, (const unsigned char *)k_method_key, k_method_key_len); if (key_pos == INVALID_POS) { diff --git a/pkg/components/ebpf/gotracer/gotracer.go b/pkg/components/ebpf/gotracer/gotracer.go index 2581ae00d7..ec23760ea3 100644 --- a/pkg/components/ebpf/gotracer/gotracer.go +++ b/pkg/components/ebpf/gotracer/gotracer.go @@ -86,7 +86,22 @@ func (p *Tracer) Load() (*ebpf.CollectionSpec, error) { return loader() } -func (p *Tracer) SetupTailCalls() {} +func (p *Tracer) SetupTailCalls() { + for _, tc := range []struct { + index int + prog *ebpf.Program + }{ + { + index: 0, + prog: p.bpfObjects.BeylaReadJsonrpcMethod, + }, + } { + err := p.bpfObjects.JsonrpcJumpTable.Update(uint32(tc.index), uint32(tc.prog.FD()), ebpf.UpdateAny) + if err != nil { + p.log.Error("error loading info tail call jump table", "error", err) + } + } +} func (p *Tracer) Constants() map[string]any { blackBoxCP := uint32(0) From 706e0c0ba8c082d0f73178047d510182f8701216 Mon Sep 17 00:00:00 2001 From: titaneric Date: Sat, 21 Jun 2025 04:23:08 +0000 Subject: [PATCH 13/40] detect json content type in ServeHTTP --- bpf/gotracer/go_nethttp.c | 36 +++++++++++++++++++----------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/bpf/gotracer/go_nethttp.c b/bpf/gotracer/go_nethttp.c index 66cf0078ec..8875088681 100644 --- a/bpf/gotracer/go_nethttp.c +++ b/bpf/gotracer/go_nethttp.c @@ -68,9 +68,10 @@ typedef struct server_http_func_invocation { u64 response_length; u64 status; tp_info_t tp; - unsigned char method[METHOD_MAX_LEN]; - unsigned char path[PATH_MAX_LEN]; - u8 _pad[5]; + u8 method[METHOD_MAX_LEN]; + u8 path[PATH_MAX_LEN]; + u8 json_content_type; + u8 _pad[4]; u64 body_addr; // pointer to the body buffer u8 content_type[HTTP_CONTENT_TYPE_MAX_LEN]; } server_http_func_invocation_t; @@ -149,9 +150,13 @@ int beyla_uprobe_ServeHTTP(struct pt_regs *ctx) { // get content-type from readContinuedLineSlice if (header_inv && header_inv->content_type[0]) { bpf_dbg_printk("Found content type in ongoing request: %s", header_inv->content_type); - __builtin_memcpy(invocation.content_type, - header_inv->content_type, - sizeof(header_inv->content_type)); + // __builtin_memcpy(invocation.content_type, + // header_inv->content_type, + // sizeof(header_inv->content_type)); + if (is_json_content_type((void *)header_inv->content_type, + sizeof(header_inv->content_type))) { + invocation.json_content_type = 1; + } } // Get method from Request.Method @@ -1294,7 +1299,6 @@ int beyla_uprobe_bodyRead(struct pt_regs *ctx) { server_http_func_invocation_t *invocation = bpf_map_lookup_elem(&ongoing_http_server_requests, &g_key); if (!invocation) { - bpf_dbg_printk("can't find invocation info for server call"); return 0; } invocation->body_addr = body_addr; @@ -1310,26 +1314,24 @@ int beyla_uprobe_bodyReadReturn(struct pt_regs *ctx) { go_addr_key_from_id(&g_key, goroutine_addr); u64 n = (u64)GO_PARAM1(ctx); + bpf_dbg_printk("n is %llu", n); server_http_func_invocation_t *invocation = bpf_map_lookup_elem(&ongoing_http_server_requests, &g_key); if (!invocation) { - bpf_dbg_printk("can't find invocation info for server call"); return 0; } - // content-type is set in invocation in ServeHTTP - bpf_dbg_printk("n is %llu", n); - bpf_dbg_printk("content type is %s", invocation->content_type); - if (n <= 0 || !invocation->body_addr) { return 0; } - unsigned char *body_buf = temp_body_mem(); - if (!body_buf) { + // json_content-type is set in invocation in ServeHTTP + if (invocation->json_content_type != 1) { + bpf_dbg_printk("content type is not json, skipping"); return 0; } - if (!is_json_content_type((void *)invocation->content_type, sizeof(invocation->content_type))) { - bpf_dbg_printk("content type is not json, skipping"); + + unsigned char *body_buf = temp_body_mem(); + if (!body_buf) { return 0; } @@ -1338,7 +1340,7 @@ int beyla_uprobe_bodyReadReturn(struct pt_regs *ctx) { bpf_dbg_printk("failed to read body, n=%llu, body_addr=%llx", n, invocation->body_addr); return 0; } - bpf_dbg_printk("body is %s", body_buf); + // bpf_dbg_printk("body is %s", body_buf); bpf_tail_call(ctx, &jsonrpc_jump_table, k_tail_jsonrpc); return 0; } From 199b7e4e002a3879df3a62f7645a1ace1365f7f2 Mon Sep 17 00:00:00 2001 From: titaneric Date: Sat, 21 Jun 2025 05:20:56 +0000 Subject: [PATCH 14/40] use `bpf_memicmp` instead of `json_str_value` to validate jsonrpc value --- bpf/gotracer/protocol_jsonrpc.h | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/bpf/gotracer/protocol_jsonrpc.h b/bpf/gotracer/protocol_jsonrpc.h index c236172c97..7105b6b7cd 100644 --- a/bpf/gotracer/protocol_jsonrpc.h +++ b/bpf/gotracer/protocol_jsonrpc.h @@ -105,17 +105,12 @@ static __always_inline u32 is_jsonrpc2_body(const unsigned char *body, u32 body_ bpf_dbg_printk("Found JSON-RPC 2.0 key"); u32 val_search_start = json_value_offset(body, body_len, key_pos + k_jsonrpc_key_len); - // The jsonrpc value should be a string - if (val_search_start >= body_len || body[val_search_start] != '"') { + if (val_search_start >= body_len) { return 0; } - u32 val_pos = json_str_value(body + val_search_start, - body_len - val_search_start, - (const unsigned char *)k_jsonrpc_val, - k_jsonrpc_val_len); - // The jsonrpc value should start immediately after the opening quote - if (val_pos == INVALID_POS || val_pos != 0) { + if (bpf_memicmp((const char *)(body + val_search_start), k_jsonrpc_val, k_jsonrpc_val_len) != + 0) { return 0; } From e427d2ed854f5d212aca96fdd6dc69be7d4a0877 Mon Sep 17 00:00:00 2001 From: titaneric Date: Sat, 21 Jun 2025 06:03:58 +0000 Subject: [PATCH 15/40] add early bound check and json_value_eq helper function --- bpf/gotracer/go_nethttp.c | 4 ++-- bpf/gotracer/protocol_jsonrpc.h | 34 ++++++++++++++++++++++++--------- 2 files changed, 27 insertions(+), 11 deletions(-) diff --git a/bpf/gotracer/go_nethttp.c b/bpf/gotracer/go_nethttp.c index 8875088681..632fc0ad62 100644 --- a/bpf/gotracer/go_nethttp.c +++ b/bpf/gotracer/go_nethttp.c @@ -1369,8 +1369,8 @@ int beyla_read_jsonrpc_method(struct pt_regs *ctx) { __builtin_memcpy(body_buf, body_map_buf, sizeof(body_buf)); if (is_jsonrpc2_body((const unsigned char *)body_buf, sizeof(body_buf))) { unsigned char method_buf[JSONRPC_METHOD_BUF_SIZE] = {}; - u32 method_len = - extract_jsonrpc2_method((const unsigned char *)body_buf, sizeof(body_buf), method_buf); + u32 method_len = extract_jsonrpc2_method( + (const unsigned char *)body_buf, sizeof(body_buf), method_buf, sizeof(method_buf)); if (method_len > 0) { bpf_dbg_printk("JSON-RPC method: %s", method_buf); read_go_str_n("JSON-RPC method", diff --git a/bpf/gotracer/protocol_jsonrpc.h b/bpf/gotracer/protocol_jsonrpc.h index 7105b6b7cd..6f5052ccd0 100644 --- a/bpf/gotracer/protocol_jsonrpc.h +++ b/bpf/gotracer/protocol_jsonrpc.h @@ -65,6 +65,13 @@ static __always_inline u32 json_str_value_end(const unsigned char *body, '"'); } +// Compares a JSON value at start with a given value. +// Returns 1 if equal, 0 otherwise. +static __always_inline u8 json_value_eq(const char *start, const char *val, u32 val_len) { + + return bpf_memicmp(start, val, val_len) == 0; +} + /** * Copies a JSON string value from body[value_start..value_end) into dest_buf. * Ensures null-termination and does not exceed dest_buf_size. @@ -104,13 +111,19 @@ static __always_inline u32 is_jsonrpc2_body(const unsigned char *body, u32 body_ bpf_dbg_printk("Found JSON-RPC 2.0 key"); - u32 val_search_start = json_value_offset(body, body_len, key_pos + k_jsonrpc_key_len); + u32 val_search_start = key_pos + k_jsonrpc_key_len; + if (val_search_start >= body_len) { + return 0; + } + + val_search_start = json_value_offset(body, body_len, key_pos + k_jsonrpc_key_len); if (val_search_start >= body_len) { return 0; } - if (bpf_memicmp((const char *)(body + val_search_start), k_jsonrpc_val, k_jsonrpc_val_len) != - 0) { + if (!json_value_eq((const char *)(body + val_search_start), + (const char *)k_jsonrpc_val, + k_jsonrpc_val_len)) { return 0; } @@ -124,7 +137,8 @@ static __always_inline u32 is_jsonrpc2_body(const unsigned char *body, u32 body_ // method_buf must be at least method_buf_len bytes. static __always_inline u32 extract_jsonrpc2_method(const unsigned char *body, u32 body_len, - unsigned char *method_buf) { + unsigned char *method_buf, + u32 method_buf_len) { u32 key_pos = json_str_value(body, body_len, (const unsigned char *)k_method_key, k_method_key_len); if (key_pos == INVALID_POS) { @@ -133,18 +147,20 @@ static __always_inline u32 extract_jsonrpc2_method(const unsigned char *body, bpf_dbg_printk("Found JSON-RPC method key"); - u32 val_search_start = json_value_offset(body, body_len, key_pos + k_method_key_len); + u32 val_search_start = key_pos + k_method_key_len; + if (val_search_start >= body_len) { + return 0; + } + + val_search_start = json_value_offset(body, body_len, key_pos + k_method_key_len); // method value should be a string if (val_search_start >= body_len || body[val_search_start] != '"') { return 0; } - bpf_dbg_printk("Found JSON-RPC method value opening quote"); - // Copy the method value from the body after the opening quote u32 value_start = val_search_start + 1; u32 value_end = json_str_value_end(body, body_len, value_start); - return copy_json_string_value( - body, value_start, value_end, method_buf, JSONRPC_METHOD_BUF_SIZE); + return copy_json_string_value(body, value_start, value_end, method_buf, method_buf_len); } \ No newline at end of file From 1d1e93b0c7ef66b7b85a02b1bad2204243fa407a Mon Sep 17 00:00:00 2001 From: titaneric Date: Sat, 21 Jun 2025 08:15:05 +0000 Subject: [PATCH 16/40] Add bound check and rewrite method value extraction --- bpf/common/tc_common.h | 7 ++++++ bpf/gotracer/go_nethttp.c | 11 ++++----- bpf/gotracer/protocol_jsonrpc.h | 42 +++++++++++++++++++++++++++------ 3 files changed, 46 insertions(+), 14 deletions(-) diff --git a/bpf/common/tc_common.h b/bpf/common/tc_common.h index bad51a47af..97400da8d8 100644 --- a/bpf/common/tc_common.h +++ b/bpf/common/tc_common.h @@ -141,6 +141,9 @@ static __always_inline u32 bpf_memstr(const unsigned char *haystack, return INVALID_POS; } for (u32 i = 0; i <= haystack_len - needle_len; i++) { + if (i + needle_len > haystack_len) { + return INVALID_POS; + } u8 found = 1; #pragma unroll // max needle length @@ -148,6 +151,10 @@ static __always_inline u32 bpf_memstr(const unsigned char *haystack, if (j >= needle_len) { break; } + if (i + j >= haystack_len) { + found = 0; + break; + } if (haystack[i + j] != needle[j]) { found = 0; break; diff --git a/bpf/gotracer/go_nethttp.c b/bpf/gotracer/go_nethttp.c index 632fc0ad62..29d4022617 100644 --- a/bpf/gotracer/go_nethttp.c +++ b/bpf/gotracer/go_nethttp.c @@ -1360,17 +1360,14 @@ int beyla_read_jsonrpc_method(struct pt_regs *ctx) { return 0; } - unsigned char *body_map_buf = temp_body_mem(); - if (!body_map_buf) { + unsigned char *body_buf = temp_body_mem(); + if (!body_buf) { return 0; } - // TODO: save this body buffer? - unsigned char body_buf[HTTP_BODY_MAX_LEN] = {}; - __builtin_memcpy(body_buf, body_map_buf, sizeof(body_buf)); - if (is_jsonrpc2_body((const unsigned char *)body_buf, sizeof(body_buf))) { + if (is_jsonrpc2_body((const unsigned char *)body_buf, HTTP_BODY_MAX_LEN)) { unsigned char method_buf[JSONRPC_METHOD_BUF_SIZE] = {}; u32 method_len = extract_jsonrpc2_method( - (const unsigned char *)body_buf, sizeof(body_buf), method_buf, sizeof(method_buf)); + (const unsigned char *)body_buf, HTTP_BODY_MAX_LEN, method_buf, sizeof(method_buf)); if (method_len > 0) { bpf_dbg_printk("JSON-RPC method: %s", method_buf); read_go_str_n("JSON-RPC method", diff --git a/bpf/gotracer/protocol_jsonrpc.h b/bpf/gotracer/protocol_jsonrpc.h index 6f5052ccd0..bdfebf3264 100644 --- a/bpf/gotracer/protocol_jsonrpc.h +++ b/bpf/gotracer/protocol_jsonrpc.h @@ -101,6 +101,34 @@ static __always_inline u32 copy_json_string_value(const unsigned char *body, return value_len; } +// Extracts a JSON string value starting at a given position (after the opening quote). +// Copies up to buf_len-1 bytes into buf, null-terminated. +// Returns the number of bytes copied (not including null terminator), or 0 on error. +static __always_inline u32 extract_json_string( + const unsigned char *body, u32 body_len, u32 value_start, unsigned char *buf, u32 buf_len) { + if (value_start >= body_len || buf_len == 0) + return 0; + + u32 value_end = value_start; + while (value_end < body_len && body[value_end] != '"') { + value_end++; + } + u32 value_len = value_end - value_start; + if (value_len == 0) + return 0; + + u32 copy_len = value_len < (buf_len - 1) ? value_len : (buf_len - 1); + + // #pragma unroll + for (u32 i = 0; i < JSONRPC_METHOD_BUF_SIZE; i++) { + if (i >= copy_len) + break; + buf[i] = body[value_start + i]; + } + buf[copy_len] = '\0'; + return copy_len; +} + // Looks for '"jsonrpc":"2.0"' static __always_inline u32 is_jsonrpc2_body(const unsigned char *body, u32 body_len) { u32 key_pos = @@ -116,11 +144,15 @@ static __always_inline u32 is_jsonrpc2_body(const unsigned char *body, u32 body_ return 0; } - val_search_start = json_value_offset(body, body_len, key_pos + k_jsonrpc_key_len); + val_search_start = json_value_offset(body, body_len, val_search_start); if (val_search_start >= body_len) { return 0; } + if (val_search_start + k_jsonrpc_val_len >= body_len) { + return 0; + } + if (!json_value_eq((const char *)(body + val_search_start), (const char *)k_jsonrpc_val, k_jsonrpc_val_len)) { @@ -152,15 +184,11 @@ static __always_inline u32 extract_jsonrpc2_method(const unsigned char *body, return 0; } - val_search_start = json_value_offset(body, body_len, key_pos + k_method_key_len); + val_search_start = json_value_offset(body, body_len, val_search_start); // method value should be a string if (val_search_start >= body_len || body[val_search_start] != '"') { return 0; } - - // Copy the method value from the body after the opening quote u32 value_start = val_search_start + 1; - u32 value_end = json_str_value_end(body, body_len, value_start); - - return copy_json_string_value(body, value_start, value_end, method_buf, method_buf_len); + return extract_json_string(body, body_len, value_start, method_buf, method_buf_len); } \ No newline at end of file From d735845962ffbd61e4fdb19980a87ad4429e2280 Mon Sep 17 00:00:00 2001 From: titaneric Date: Sat, 21 Jun 2025 09:55:49 +0000 Subject: [PATCH 17/40] opening bracket check in extract_json_string --- bpf/gotracer/protocol_jsonrpc.h | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/bpf/gotracer/protocol_jsonrpc.h b/bpf/gotracer/protocol_jsonrpc.h index bdfebf3264..f437cfbc2f 100644 --- a/bpf/gotracer/protocol_jsonrpc.h +++ b/bpf/gotracer/protocol_jsonrpc.h @@ -11,7 +11,7 @@ static const u32 k_application_json_len = sizeof(k_application_json) - 1; static const char k_method_key[] = "\"method\""; static const u32 k_method_key_len = sizeof(k_method_key) - 1; -enum { JSONRPC_METHOD_BUF_SIZE = 16 }; +enum { JSON_MAX_STRING_LEN = 16, JSONRPC_METHOD_BUF_SIZE = 16 }; // should match application/json, application/json-rpc, application/jsonrequest // listed in https://www.jsonrpc.org/historical/json-rpc-over-http.html @@ -101,29 +101,35 @@ static __always_inline u32 copy_json_string_value(const unsigned char *body, return value_len; } -// Extracts a JSON string value starting at a given position (after the opening quote). +// Extracts a JSON string value starting at a given position. // Copies up to buf_len-1 bytes into buf, null-terminated. // Returns the number of bytes copied (not including null terminator), or 0 on error. static __always_inline u32 extract_json_string( const unsigned char *body, u32 body_len, u32 value_start, unsigned char *buf, u32 buf_len) { - if (value_start >= body_len || buf_len == 0) + if (value_start >= body_len || buf_len == 0) { return 0; + } + + if (body[value_start] != '"') { + return 0; + } - u32 value_end = value_start; + const u32 str_start = value_start + 1; + u32 value_end = str_start; while (value_end < body_len && body[value_end] != '"') { value_end++; } - u32 value_len = value_end - value_start; + const u32 value_len = value_end - str_start; if (value_len == 0) return 0; - u32 copy_len = value_len < (buf_len - 1) ? value_len : (buf_len - 1); + const u32 copy_len = value_len < (buf_len - 1) ? value_len : (buf_len - 1); // #pragma unroll - for (u32 i = 0; i < JSONRPC_METHOD_BUF_SIZE; i++) { + for (u32 i = 0; i < buf_len; i++) { if (i >= copy_len) break; - buf[i] = body[value_start + i]; + buf[i] = body[str_start + i]; } buf[copy_len] = '\0'; return copy_len; @@ -185,10 +191,8 @@ static __always_inline u32 extract_jsonrpc2_method(const unsigned char *body, } val_search_start = json_value_offset(body, body_len, val_search_start); - // method value should be a string - if (val_search_start >= body_len || body[val_search_start] != '"') { + if (val_search_start >= body_len) { return 0; } - u32 value_start = val_search_start + 1; - return extract_json_string(body, body_len, value_start, method_buf, method_buf_len); + return extract_json_string(body, body_len, val_search_start, method_buf, method_buf_len); } \ No newline at end of file From df327ca1cae17a054ad16d1cd76e47dda8d6f174 Mon Sep 17 00:00:00 2001 From: titaneric Date: Sat, 21 Jun 2025 12:41:48 +0000 Subject: [PATCH 18/40] tidy up jsonrpc parser --- bpf/gotracer/protocol_jsonrpc.h | 46 +++------------------------------ 1 file changed, 4 insertions(+), 42 deletions(-) diff --git a/bpf/gotracer/protocol_jsonrpc.h b/bpf/gotracer/protocol_jsonrpc.h index f437cfbc2f..68c56c0a0a 100644 --- a/bpf/gotracer/protocol_jsonrpc.h +++ b/bpf/gotracer/protocol_jsonrpc.h @@ -54,17 +54,6 @@ static __always_inline u32 json_str_value(const unsigned char *body, return bpf_memstr(body, body_len, str, str_len); } -// Returns the end position (index of closing quote) of a JSON string value. -// If not found, returns body_len. -static __always_inline u32 json_str_value_end(const unsigned char *body, - u32 body_len, - u32 value_start) { - // find_first_pos_of expects unsigned char*, so cast accordingly - return value_start + find_first_pos_of((unsigned char *)(body + value_start), - (unsigned char *)(body + body_len), - '"'); -} - // Compares a JSON value at start with a given value. // Returns 1 if equal, 0 otherwise. static __always_inline u8 json_value_eq(const char *start, const char *val, u32 val_len) { @@ -72,35 +61,6 @@ static __always_inline u8 json_value_eq(const char *start, const char *val, u32 return bpf_memicmp(start, val, val_len) == 0; } -/** - * Copies a JSON string value from body[value_start..value_end) into dest_buf. - * Ensures null-termination and does not exceed dest_buf_size. - * Returns the number of bytes copied (excluding null terminator), or 0 on error. - */ -static __always_inline u32 copy_json_string_value(const unsigned char *body, - u32 value_start, - u32 value_end, - unsigned char *dest_buf, - u32 dest_buf_size) { - u32 value_len = value_end - value_start; - if (value_len <= 0) { - return 0; - } - if (value_len >= dest_buf_size) { - value_len = dest_buf_size - 1; // leave space for null terminator - } - -#pragma unroll - for (u32 i = 0; i < dest_buf_size; i++) { - if (i >= value_len) { - break; - } - dest_buf[i] = body[value_start + i]; - } - dest_buf[value_len] = '\0'; - return value_len; -} - // Extracts a JSON string value starting at a given position. // Copies up to buf_len-1 bytes into buf, null-terminated. // Returns the number of bytes copied (not including null terminator), or 0 on error. @@ -120,15 +80,17 @@ static __always_inline u32 extract_json_string( value_end++; } const u32 value_len = value_end - str_start; - if (value_len == 0) + if (value_len == 0) { return 0; + } const u32 copy_len = value_len < (buf_len - 1) ? value_len : (buf_len - 1); // #pragma unroll for (u32 i = 0; i < buf_len; i++) { - if (i >= copy_len) + if (i >= copy_len) { break; + } buf[i] = body[str_start + i]; } buf[copy_len] = '\0'; From 5d0b7ff6bbe116947ab5b0efa4fd17e185b7e1a4 Mon Sep 17 00:00:00 2001 From: titaneric Date: Sat, 21 Jun 2025 13:06:45 +0000 Subject: [PATCH 19/40] reorder field in `server_http_func_invocation` struct --- bpf/common/common.h | 1 - bpf/gotracer/go_nethttp.c | 8 +++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/bpf/common/common.h b/bpf/common/common.h index 5ef8fce924..cd0ede1463 100644 --- a/bpf/common/common.h +++ b/bpf/common/common.h @@ -32,7 +32,6 @@ #define SCHEME_MAX_LEN 10 #define HTTP_BODY_MAX_LEN 64 #define HTTP_HEADER_MAX_LEN 100 -#define CONTENT_TYPE_KEY_LEN 12 // "content-type" length #define HTTP_CONTENT_TYPE_MAX_LEN 16 // Trace of an HTTP call invocation. It is instantiated by the return uprobe and forwarded to the diff --git a/bpf/gotracer/go_nethttp.c b/bpf/gotracer/go_nethttp.c index 29d4022617..ce9d8ec9cb 100644 --- a/bpf/gotracer/go_nethttp.c +++ b/bpf/gotracer/go_nethttp.c @@ -67,13 +67,13 @@ typedef struct server_http_func_invocation { u64 content_length; u64 response_length; u64 status; + u64 body_addr; // pointer to the body buffer tp_info_t tp; + unsigned char content_type[HTTP_CONTENT_TYPE_MAX_LEN]; u8 method[METHOD_MAX_LEN]; u8 path[PATH_MAX_LEN]; u8 json_content_type; - u8 _pad[4]; - u64 body_addr; // pointer to the body buffer - u8 content_type[HTTP_CONTENT_TYPE_MAX_LEN]; + u8 _pad[12]; } server_http_func_invocation_t; struct { @@ -1360,6 +1360,8 @@ int beyla_read_jsonrpc_method(struct pt_regs *ctx) { return 0; } + // tail call is guaranteed to run on the same CPU as its caller + // so we can shared the same buffer via a per-CPU map unsigned char *body_buf = temp_body_mem(); if (!body_buf) { return 0; From c9c563ab8ccff1ff74f65743f1da454d046c41a8 Mon Sep 17 00:00:00 2001 From: titaneric Date: Sat, 21 Jun 2025 16:19:34 +0000 Subject: [PATCH 20/40] Add jsonrpc test server --- .../components/testserver/jsonrpc/jsonrpc.go | 64 +++++++++++++++++++ .../components/testserver/testserver.go | 7 ++ test/integration/docker-compose.yml | 1 + 3 files changed, 72 insertions(+) create mode 100644 test/integration/components/testserver/jsonrpc/jsonrpc.go diff --git a/test/integration/components/testserver/jsonrpc/jsonrpc.go b/test/integration/components/testserver/jsonrpc/jsonrpc.go new file mode 100644 index 0000000000..ae48adb862 --- /dev/null +++ b/test/integration/components/testserver/jsonrpc/jsonrpc.go @@ -0,0 +1,64 @@ +package jsonrpc + +import ( + "fmt" + "io" + "log/slog" + "net/http" + "net/rpc" + "net/rpc/jsonrpc" +) + +// Args defines the arguments for the RPC methods. +type Args struct { + A, B int +} + +// Arith provides methods for arithmetic operations. +type Arith struct{} + +// Multiply multiplies two numbers and returns the result. +func (t *Arith) Multiply(args *Args, reply *int) error { + *reply = args.A * args.B + return nil +} + +// ReadWriteCloserWrapper wraps an io.Reader and io.Writer to implement io.ReadWriteCloser. +type ReadWriteCloserWrapper struct { + io.Reader + io.Writer +} + +// Close is a no-op to satisfy the io.ReadWriteCloser interface. +func (w *ReadWriteCloserWrapper) Close() error { + return nil +} + +func Setup(port int) { + arith := new(Arith) + rpc.Register(arith) + + log := slog.With("component", "jsonrpc.Server") + address := fmt.Sprintf(":%d", port) + log.Info("starting JSON-RPC server", "address", address) + err := http.ListenAndServe(address, HTTPHandler(log, port)) + log.Error("JSON-RPC server has unexpectedly stopped", "error", err) +} + +func HTTPHandler(log *slog.Logger, echoPort int) http.HandlerFunc { + return func(rw http.ResponseWriter, req *http.Request) { + log.Debug("received request", "url", req.RequestURI) + if req.RequestURI == "/jsonrpc" { + if req.Method != http.MethodPost { + http.Error(rw, "Only POST method is allowed", http.StatusMethodNotAllowed) + return + } + // Wrap the request body and response writer in a ReadWriteCloser. + conn := &ReadWriteCloserWrapper{Reader: req.Body, Writer: rw} + // Serve the request using JSON-RPC codec. + rpc.ServeCodec(jsonrpc.NewServerCodec(conn)) + } else { + http.NotFound(rw, req) + } + } +} diff --git a/test/integration/components/testserver/testserver.go b/test/integration/components/testserver/testserver.go index c6c64949c3..62c9a7c096 100644 --- a/test/integration/components/testserver/testserver.go +++ b/test/integration/components/testserver/testserver.go @@ -12,6 +12,7 @@ import ( "github.com/open-telemetry/opentelemetry-ebpf-instrumentation/test/integration/components/testserver/gorillamid" "github.com/open-telemetry/opentelemetry-ebpf-instrumentation/test/integration/components/testserver/gorillamid2" grpctest "github.com/open-telemetry/opentelemetry-ebpf-instrumentation/test/integration/components/testserver/grpc/server" + "github.com/open-telemetry/opentelemetry-ebpf-instrumentation/test/integration/components/testserver/jsonrpc" "github.com/open-telemetry/opentelemetry-ebpf-instrumentation/test/integration/components/testserver/std" ) @@ -35,6 +36,7 @@ type config struct { GorillaMid2Port int `env:"GORILLA_MID2_PORT" envDefault:"8087"` GRPCPort int `env:"GRPC_PORT" envDefault:"5051"` GRPCTLSPort int `env:"GRPC_TLS_PORT" envDefault:"50051"` + JSONRPCPort int `env:"JSRONRPC_PORT" envDefault:"8088"` LogLevel string `env:"LOG_LEVEL" envDefault:"INFO"` } @@ -88,6 +90,11 @@ func main() { close(wait) }() + go func() { + jsonrpc.Setup(cfg.JSONRPCPort) + close(wait) + }() + // wait indefinitely unless any server crashes <-wait slog.Warn("stopping process") diff --git a/test/integration/docker-compose.yml b/test/integration/docker-compose.yml index 72ab949487..c21ef5b4a5 100644 --- a/test/integration/docker-compose.yml +++ b/test/integration/docker-compose.yml @@ -12,6 +12,7 @@ services: - "8082:8082" - "8083:8083" - "8087:8087" + - "8088:8088" - "8383:8383" - "5051:5051" - "50051:50051" From 173f47e688572f27ecca4ddb8ce98f33d85fb390 Mon Sep 17 00:00:00 2001 From: titaneric Date: Sat, 21 Jun 2025 17:37:18 +0000 Subject: [PATCH 21/40] add basic jsonrpc integration test --- .../components/testserver/jsonrpc/jsonrpc.go | 3 +- test/integration/red_test.go | 45 +++++++++++++++++++ test/integration/suites_test.go | 19 ++++---- 3 files changed, 57 insertions(+), 10 deletions(-) diff --git a/test/integration/components/testserver/jsonrpc/jsonrpc.go b/test/integration/components/testserver/jsonrpc/jsonrpc.go index ae48adb862..1f687a705d 100644 --- a/test/integration/components/testserver/jsonrpc/jsonrpc.go +++ b/test/integration/components/testserver/jsonrpc/jsonrpc.go @@ -58,7 +58,8 @@ func HTTPHandler(log *slog.Logger, echoPort int) http.HandlerFunc { // Serve the request using JSON-RPC codec. rpc.ServeCodec(jsonrpc.NewServerCodec(conn)) } else { - http.NotFound(rw, req) + rw.WriteHeader(http.StatusOK) } + } } diff --git a/test/integration/red_test.go b/test/integration/red_test.go index fd8c2fa60a..bb4fdfbf8d 100644 --- a/test/integration/red_test.go +++ b/test/integration/red_test.go @@ -28,6 +28,7 @@ const ( instrumentedServiceGorillaMidURL = "http://localhost:8083" instrumentedServiceGorillaMid2URL = "http://localhost:8087" instrumentedServiceStdTLSURL = "https://localhost:8383" + instrumentedServiceJsonRpcURL = "http://localhost:8088" prometheusHostPort = "localhost:9090" jaegerQueryURL = "http://localhost:16686/api/traces" @@ -93,6 +94,18 @@ func testREDMetricsShortHTTP(t *testing.T) { } } +func testREDMetricsJSONRPCHTTP(t *testing.T) { + for _, testCaseURL := range []string{ + instrumentedServiceJsonRpcURL, + } { + t.Run(testCaseURL, func(t *testing.T) { + waitForTestComponents(t, testCaseURL) + testREDMetricsForJSONRPCHTTP(t, testCaseURL, "testserver", "integration-test") + // testSpanMetricsForHTTPLibraryOTelFormat(t, "testserver", "integration-test") + }) + } +} + func testExemplarsExist(t *testing.T) { url := "http://" + prometheusHostPort + "/api/v1/query_exemplars?query=http_server_request_duration_seconds_bucket" @@ -261,6 +274,38 @@ func testServiceGraphMetricsForHTTPLibrary(t *testing.T, svcNs string) { assert.Equal(t, 0, val) } +func testREDMetricsForJSONRPCHTTP(t *testing.T, url, svcName, svcNs string) { + urlPath := "/jsonrpc" + jsonBody := `{"jsonrpc":"2.0","method":"Arith.Multiply","params":[{"A":7,"B":8}],"id":1}` + + for i := 0; i < 4; i++ { + doHTTPPost(t, url+urlPath, 200, []byte(jsonBody)) + } + + // Eventually, Prometheus would make this query visible + pq := prom.Client{HostPort: prometheusHostPort} + var results []prom.Result + test.Eventually(t, testTimeout, func(t require.TestingT) { + var err error + results, err = pq.Query(`http_server_request_duration_seconds_count{` + + `http_request_method="Arith.M",` + + `http_response_status_code="200",` + + `service_namespace="` + svcNs + `",` + + `service_name="` + svcName + `",` + + `url_path="` + urlPath + `"}`) + require.NoError(t, err) + enoughPromResults(t, results) + val := totalPromCount(t, results) + assert.LessOrEqual(t, 3, val) + if len(results) > 0 { + res := results[0] + addr := res.Metric["client_address"] + assert.NotNil(t, addr) + } + }) + +} + func testREDMetricsForHTTPLibrary(t *testing.T, url, svcName, svcNs string) { path := "/basic/" + rndStr() diff --git a/test/integration/suites_test.go b/test/integration/suites_test.go index 4b6e6c0456..3506445f5c 100644 --- a/test/integration/suites_test.go +++ b/test/integration/suites_test.go @@ -25,15 +25,16 @@ func TestSuite(t *testing.T) { compose, err := docker.ComposeSuite("docker-compose.yml", path.Join(pathOutput, "test-suite.log")) require.NoError(t, err) require.NoError(t, compose.Up()) - t.Run("RED metrics", testREDMetricsHTTP) - t.Run("HTTP traces", testHTTPTraces) - t.Run("HTTP traces (no traceID)", testHTTPTracesNoTraceID) - t.Run("GRPC traces", testGRPCTraces) - t.Run("GRPC RED metrics", testREDMetricsGRPC) - t.Run("GRPC TLS RED metrics", testREDMetricsGRPCTLS) - t.Run("Internal Prometheus metrics", testInternalPrometheusExport) - t.Run("Exemplars exist", testExemplarsExist) - t.Run("Testing Host Info metric", testHostInfo) + // t.Run("RED metrics", testREDMetricsHTTP) + t.Run("RED JSON RPC metrics", testREDMetricsJSONRPCHTTP) + // t.Run("HTTP traces", testHTTPTraces) + // t.Run("HTTP traces (no traceID)", testHTTPTracesNoTraceID) + // t.Run("GRPC traces", testGRPCTraces) + // t.Run("GRPC RED metrics", testREDMetricsGRPC) + // t.Run("GRPC TLS RED metrics", testREDMetricsGRPCTLS) + // t.Run("Internal Prometheus metrics", testInternalPrometheusExport) + // t.Run("Exemplars exist", testExemplarsExist) + // t.Run("Testing Host Info metric", testHostInfo) require.NoError(t, compose.Close()) } From 2dfff52544ad88aa6311affcd997282b682861c5 Mon Sep 17 00:00:00 2001 From: titaneric Date: Sun, 22 Jun 2025 06:38:53 +0000 Subject: [PATCH 22/40] move test json body into file --- .../components/testserver/jsonrpc/body/formated.json | 11 +++++++++++ test/integration/red_test.go | 8 ++++++-- 2 files changed, 17 insertions(+), 2 deletions(-) create mode 100644 test/integration/components/testserver/jsonrpc/body/formated.json diff --git a/test/integration/components/testserver/jsonrpc/body/formated.json b/test/integration/components/testserver/jsonrpc/body/formated.json new file mode 100644 index 0000000000..575e293bdf --- /dev/null +++ b/test/integration/components/testserver/jsonrpc/body/formated.json @@ -0,0 +1,11 @@ +{ + "jsonrpc": "2.0", + "method": "Arith.Multiply", + "params": [ + { + "A": 7, + "B": 8 + } + ], + "id": 1 +} \ No newline at end of file diff --git a/test/integration/red_test.go b/test/integration/red_test.go index bb4fdfbf8d..59121ff037 100644 --- a/test/integration/red_test.go +++ b/test/integration/red_test.go @@ -8,6 +8,8 @@ import ( "io" "math/rand/v2" "net/http" + "os" + "path" "strconv" "strings" "testing" @@ -275,8 +277,10 @@ func testServiceGraphMetricsForHTTPLibrary(t *testing.T, svcNs string) { } func testREDMetricsForJSONRPCHTTP(t *testing.T, url, svcName, svcNs string) { + jsonBody, err := os.ReadFile(path.Join(pathRoot, "test", "integration", "components", "testserver", "jsonrpc", "body", "formated.json")) + require.NoError(t, err) urlPath := "/jsonrpc" - jsonBody := `{"jsonrpc":"2.0","method":"Arith.Multiply","params":[{"A":7,"B":8}],"id":1}` + expectedMethod := "Arith.M" for i := 0; i < 4; i++ { doHTTPPost(t, url+urlPath, 200, []byte(jsonBody)) @@ -288,7 +292,7 @@ func testREDMetricsForJSONRPCHTTP(t *testing.T, url, svcName, svcNs string) { test.Eventually(t, testTimeout, func(t require.TestingT) { var err error results, err = pq.Query(`http_server_request_duration_seconds_count{` + - `http_request_method="Arith.M",` + + `http_request_method="` + expectedMethod + `",` + `http_response_status_code="200",` + `service_namespace="` + svcNs + `",` + `service_name="` + svcName + `",` + From 9a1a65677694ecef70615a1b714f6cc6e526e183 Mon Sep 17 00:00:00 2001 From: titaneric Date: Sun, 22 Jun 2025 06:46:41 +0000 Subject: [PATCH 23/40] restore suite test --- test/integration/suites_test.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/test/integration/suites_test.go b/test/integration/suites_test.go index 3506445f5c..9d60572a5b 100644 --- a/test/integration/suites_test.go +++ b/test/integration/suites_test.go @@ -25,16 +25,16 @@ func TestSuite(t *testing.T) { compose, err := docker.ComposeSuite("docker-compose.yml", path.Join(pathOutput, "test-suite.log")) require.NoError(t, err) require.NoError(t, compose.Up()) - // t.Run("RED metrics", testREDMetricsHTTP) + t.Run("RED metrics", testREDMetricsHTTP) t.Run("RED JSON RPC metrics", testREDMetricsJSONRPCHTTP) - // t.Run("HTTP traces", testHTTPTraces) - // t.Run("HTTP traces (no traceID)", testHTTPTracesNoTraceID) - // t.Run("GRPC traces", testGRPCTraces) - // t.Run("GRPC RED metrics", testREDMetricsGRPC) - // t.Run("GRPC TLS RED metrics", testREDMetricsGRPCTLS) - // t.Run("Internal Prometheus metrics", testInternalPrometheusExport) - // t.Run("Exemplars exist", testExemplarsExist) - // t.Run("Testing Host Info metric", testHostInfo) + t.Run("HTTP traces", testHTTPTraces) + t.Run("HTTP traces (no traceID)", testHTTPTracesNoTraceID) + t.Run("GRPC traces", testGRPCTraces) + t.Run("GRPC RED metrics", testREDMetricsGRPC) + t.Run("GRPC TLS RED metrics", testREDMetricsGRPCTLS) + t.Run("Internal Prometheus metrics", testInternalPrometheusExport) + t.Run("Exemplars exist", testExemplarsExist) + t.Run("Testing Host Info metric", testHostInfo) require.NoError(t, compose.Close()) } From 7818ca94deb0a29a560ecb3d1de292233df3cf48 Mon Sep 17 00:00:00 2001 From: titaneric Date: Sun, 22 Jun 2025 07:47:04 +0000 Subject: [PATCH 24/40] Add testSpanMetricsForJSONRPCHTTP --- test/integration/red_test.go | 56 +++++++++++++++++++++++++++++++++++- 1 file changed, 55 insertions(+), 1 deletion(-) diff --git a/test/integration/red_test.go b/test/integration/red_test.go index 59121ff037..9ad528c62b 100644 --- a/test/integration/red_test.go +++ b/test/integration/red_test.go @@ -103,7 +103,7 @@ func testREDMetricsJSONRPCHTTP(t *testing.T) { t.Run(testCaseURL, func(t *testing.T) { waitForTestComponents(t, testCaseURL) testREDMetricsForJSONRPCHTTP(t, testCaseURL, "testserver", "integration-test") - // testSpanMetricsForHTTPLibraryOTelFormat(t, "testserver", "integration-test") + testSpanMetricsForJSONRPCHTTP(t, "testserver", "integration-test") }) } } @@ -237,6 +237,60 @@ func testSpanMetricsForHTTPLibrary(t *testing.T, svcName, svcNs string) { }) } +// **IMPORTANT** Tests must first call -> func testREDMetricsForJSONRPCHTTP(t *testing.T, url, svcName, svcNs string) { +func testSpanMetricsForJSONRPCHTTP(t *testing.T, svcName, svcNs string) { + pq := prom.Client{HostPort: prometheusHostPort} + var results []prom.Result + + expectedSpanName := "Arith.M /jsonrpc" + + // Test span metrics + test.Eventually(t, testTimeout, func(t require.TestingT) { + var err error + results, err = pq.Query(`traces_span_metrics_duration_count{` + + `span_kind="SPAN_KIND_SERVER",` + + `status_code="STATUS_CODE_UNSET",` + // 404 is OK for server spans + `service_namespace="` + svcNs + `",` + + `service_name="` + svcName + `",` + + `span_name="` + expectedSpanName + `"` + + `}`) + require.NoError(t, err) + // check span metric latency exists + enoughPromResults(t, results) + val := totalPromCount(t, results) + assert.LessOrEqual(t, 3, val) + }) + + test.Eventually(t, testTimeout, func(t require.TestingT) { + var err error + results, err = pq.Query(`traces_span_metrics_calls_total{` + + `span_kind="SPAN_KIND_SERVER",` + + `status_code="STATUS_CODE_UNSET",` + // 404 is OK for server spans + `service_namespace="` + svcNs + `",` + + `service_name="` + svcName + `",` + + `span_name="` + expectedSpanName + `"` + + `}`) + require.NoError(t, err) + // check calls total exists + enoughPromResults(t, results) + val := totalPromCount(t, results) + assert.LessOrEqual(t, 3, val) + }) + + test.Eventually(t, testTimeout, func(t require.TestingT) { + var err error + results, err = pq.Query(`traces_target_info{` + + `service_namespace="` + svcNs + `",` + + `service_name="` + svcName + `",` + + `telemetry_sdk_language="go"` + + `}`) + require.NoError(t, err) + enoughPromResults(t, results) + val := totalPromCount(t, results) + assert.LessOrEqual(t, 1, val) // we report this count for each service, doesn't matter how many calls + }) +} + // **IMPORTANT** Tests must first call -> func testREDMetricsForHTTPLibrary(t *testing.T, url, svcName, svcNs string) { func testServiceGraphMetricsForHTTPLibrary(t *testing.T, svcNs string) { pq := prom.Client{HostPort: prometheusHostPort} From 1980292796750839a336fd2c7a1ab90c3eaff2f4 Mon Sep 17 00:00:00 2001 From: titaneric Date: Sun, 22 Jun 2025 08:08:56 +0000 Subject: [PATCH 25/40] Add json-rpc in multiprocess test --- test/integration/docker-compose-multiexec.yml | 3 ++- test/integration/multiprocess_test.go | 7 +++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/test/integration/docker-compose-multiexec.yml b/test/integration/docker-compose-multiexec.yml index 2381a8e52e..8719c91a6c 100644 --- a/test/integration/docker-compose-multiexec.yml +++ b/test/integration/docker-compose-multiexec.yml @@ -7,7 +7,8 @@ services: dockerfile: test/integration/components/testserver/Dockerfile image: hatest-testserver ports: - - "8080:8080" + - "8080:8080" # std http server + - "8088:8088" # jsonrpc server environment: LOG_LEVEL: DEBUG depends_on: diff --git a/test/integration/multiprocess_test.go b/test/integration/multiprocess_test.go index 3853b99160..60fc32f168 100644 --- a/test/integration/multiprocess_test.go +++ b/test/integration/multiprocess_test.go @@ -39,6 +39,13 @@ func TestMultiProcess(t *testing.T) { // it doesn't instrument too the process from the other container checkReportedOnlyOnce(t, "http://localhost:8900", "rename1") }) + t.Run("Go RED metrics: JSON RPC", func(t *testing.T) { + waitForTestComponents(t, instrumentedServiceJsonRpcURL) + testREDMetricsForJSONRPCHTTP(t, instrumentedServiceJsonRpcURL, "testserver", "initial-set") + // checks that, instrumenting the process from this container, + // it doesn't instrument too the process from the other container + checkReportedOnlyOnce(t, instrumentedServiceJsonRpcURL, "rename1") + }) t.Run("Go RED metrics: rust service ssl", func(t *testing.T) { waitForTestComponents(t, "https://localhost:8491") From 437ecb3cbe432b0926b04c14e719a4a10e40b6f6 Mon Sep 17 00:00:00 2001 From: titaneric Date: Tue, 24 Jun 2025 17:21:28 +0000 Subject: [PATCH 26/40] move headers into static instead of stack, and format --- bpf/gotracer/go_nethttp.c | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/bpf/gotracer/go_nethttp.c b/bpf/gotracer/go_nethttp.c index ce9d8ec9cb..7df7a1bed8 100644 --- a/bpf/gotracer/go_nethttp.c +++ b/bpf/gotracer/go_nethttp.c @@ -37,6 +37,8 @@ #include enum { k_tail_jsonrpc = 0 }; +static const char traceparent[] = "traceparent: "; +static const char content_type[] = "content-type: "; typedef struct http_client_data { s64 content_length; @@ -73,7 +75,7 @@ typedef struct server_http_func_invocation { u8 method[METHOD_MAX_LEN]; u8 path[PATH_MAX_LEN]; u8 json_content_type; - u8 _pad[12]; + u8 _pad[4]; } server_http_func_invocation_t; struct { @@ -94,6 +96,7 @@ static __always_inline unsigned char *temp_header_mem() { const u32 zero = 0; return bpf_map_lookup_elem(&temp_header_mem_store, &zero); } + struct { __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); __type(key, u32); @@ -150,9 +153,6 @@ int beyla_uprobe_ServeHTTP(struct pt_regs *ctx) { // get content-type from readContinuedLineSlice if (header_inv && header_inv->content_type[0]) { bpf_dbg_printk("Found content type in ongoing request: %s", header_inv->content_type); - // __builtin_memcpy(invocation.content_type, - // header_inv->content_type, - // sizeof(header_inv->content_type)); if (is_json_content_type((void *)header_inv->content_type, sizeof(header_inv->content_type))) { invocation.json_content_type = 1; @@ -374,9 +374,6 @@ int beyla_uprobe_readContinuedLineSliceReturns(struct pt_regs *ctx) { return 0; }; - const char traceparent[] = "traceparent: "; - const char content_type[] = "content-type: "; - const u32 w3c_value_start = sizeof(traceparent) - 1; const u32 content_type_value_start = sizeof(content_type) - 1; From 8c55f26ced0cff71fb1ec65135c5583aaf4f8299 Mon Sep 17 00:00:00 2001 From: titaneric Date: Tue, 24 Jun 2025 17:27:17 +0000 Subject: [PATCH 27/40] remove commented code --- bpf/gotracer/protocol_jsonrpc.h | 1 - 1 file changed, 1 deletion(-) diff --git a/bpf/gotracer/protocol_jsonrpc.h b/bpf/gotracer/protocol_jsonrpc.h index 68c56c0a0a..327f1522ad 100644 --- a/bpf/gotracer/protocol_jsonrpc.h +++ b/bpf/gotracer/protocol_jsonrpc.h @@ -86,7 +86,6 @@ static __always_inline u32 extract_json_string( const u32 copy_len = value_len < (buf_len - 1) ? value_len : (buf_len - 1); - // #pragma unroll for (u32 i = 0; i < buf_len; i++) { if (i >= copy_len) { break; From 3ec2691ceb70237eb91764876444f9af887b5562 Mon Sep 17 00:00:00 2001 From: titaneric Date: Tue, 24 Jun 2025 17:46:51 +0000 Subject: [PATCH 28/40] fix the clang tidy error --- bpf/gotracer/protocol_jsonrpc.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bpf/gotracer/protocol_jsonrpc.h b/bpf/gotracer/protocol_jsonrpc.h index 327f1522ad..0cca29b0d9 100644 --- a/bpf/gotracer/protocol_jsonrpc.h +++ b/bpf/gotracer/protocol_jsonrpc.h @@ -1,6 +1,9 @@ +#pragma once + #include #include #include +#include static const char k_jsonrpc_key[] = "\"jsonrpc\""; static const u32 k_jsonrpc_key_len = sizeof(k_jsonrpc_key) - 1; From dfb47c45f45947afc51e8f4415bcadf63c1a78fc Mon Sep 17 00:00:00 2001 From: titaneric Date: Tue, 24 Jun 2025 18:23:29 +0000 Subject: [PATCH 29/40] fix golint issue --- test/integration/components/testserver/jsonrpc/jsonrpc.go | 7 +++---- test/integration/multiprocess_test.go | 6 +++--- test/integration/red_test.go | 7 +++---- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/test/integration/components/testserver/jsonrpc/jsonrpc.go b/test/integration/components/testserver/jsonrpc/jsonrpc.go index 1f687a705d..68fb3a53b7 100644 --- a/test/integration/components/testserver/jsonrpc/jsonrpc.go +++ b/test/integration/components/testserver/jsonrpc/jsonrpc.go @@ -36,16 +36,16 @@ func (w *ReadWriteCloserWrapper) Close() error { func Setup(port int) { arith := new(Arith) - rpc.Register(arith) + _ = rpc.Register(arith) log := slog.With("component", "jsonrpc.Server") address := fmt.Sprintf(":%d", port) log.Info("starting JSON-RPC server", "address", address) - err := http.ListenAndServe(address, HTTPHandler(log, port)) + err := http.ListenAndServe(address, HTTPHandler(log)) log.Error("JSON-RPC server has unexpectedly stopped", "error", err) } -func HTTPHandler(log *slog.Logger, echoPort int) http.HandlerFunc { +func HTTPHandler(log *slog.Logger) http.HandlerFunc { return func(rw http.ResponseWriter, req *http.Request) { log.Debug("received request", "url", req.RequestURI) if req.RequestURI == "/jsonrpc" { @@ -60,6 +60,5 @@ func HTTPHandler(log *slog.Logger, echoPort int) http.HandlerFunc { } else { rw.WriteHeader(http.StatusOK) } - } } diff --git a/test/integration/multiprocess_test.go b/test/integration/multiprocess_test.go index 60fc32f168..f04bd03053 100644 --- a/test/integration/multiprocess_test.go +++ b/test/integration/multiprocess_test.go @@ -40,11 +40,11 @@ func TestMultiProcess(t *testing.T) { checkReportedOnlyOnce(t, "http://localhost:8900", "rename1") }) t.Run("Go RED metrics: JSON RPC", func(t *testing.T) { - waitForTestComponents(t, instrumentedServiceJsonRpcURL) - testREDMetricsForJSONRPCHTTP(t, instrumentedServiceJsonRpcURL, "testserver", "initial-set") + waitForTestComponents(t, instrumentedServiceJSONRPCURL) + testREDMetricsForJSONRPCHTTP(t, instrumentedServiceJSONRPCURL, "testserver", "initial-set") // checks that, instrumenting the process from this container, // it doesn't instrument too the process from the other container - checkReportedOnlyOnce(t, instrumentedServiceJsonRpcURL, "rename1") + checkReportedOnlyOnce(t, instrumentedServiceJSONRPCURL, "rename1") }) t.Run("Go RED metrics: rust service ssl", func(t *testing.T) { diff --git a/test/integration/red_test.go b/test/integration/red_test.go index 9ad528c62b..d44f6f6688 100644 --- a/test/integration/red_test.go +++ b/test/integration/red_test.go @@ -30,7 +30,7 @@ const ( instrumentedServiceGorillaMidURL = "http://localhost:8083" instrumentedServiceGorillaMid2URL = "http://localhost:8087" instrumentedServiceStdTLSURL = "https://localhost:8383" - instrumentedServiceJsonRpcURL = "http://localhost:8088" + instrumentedServiceJSONRPCURL = "http://localhost:8088" prometheusHostPort = "localhost:9090" jaegerQueryURL = "http://localhost:16686/api/traces" @@ -98,7 +98,7 @@ func testREDMetricsShortHTTP(t *testing.T) { func testREDMetricsJSONRPCHTTP(t *testing.T) { for _, testCaseURL := range []string{ - instrumentedServiceJsonRpcURL, + instrumentedServiceJSONRPCURL, } { t.Run(testCaseURL, func(t *testing.T) { waitForTestComponents(t, testCaseURL) @@ -337,7 +337,7 @@ func testREDMetricsForJSONRPCHTTP(t *testing.T, url, svcName, svcNs string) { expectedMethod := "Arith.M" for i := 0; i < 4; i++ { - doHTTPPost(t, url+urlPath, 200, []byte(jsonBody)) + doHTTPPost(t, url+urlPath, 200, jsonBody) } // Eventually, Prometheus would make this query visible @@ -361,7 +361,6 @@ func testREDMetricsForJSONRPCHTTP(t *testing.T, url, svcName, svcNs string) { assert.NotNil(t, addr) } }) - } func testREDMetricsForHTTPLibrary(t *testing.T, url, svcName, svcNs string) { From ab710b67425260864b94b18fb2ebf22c588f19ca Mon Sep 17 00:00:00 2001 From: titaneric Date: Wed, 25 Jun 2025 17:00:40 +0000 Subject: [PATCH 30/40] overwrite jsonrpc port in duplicate testserver --- test/integration/components/testserver/duplicate_testserver.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/components/testserver/duplicate_testserver.sh b/test/integration/components/testserver/duplicate_testserver.sh index 0a7a7ac9f1..47baf2af24 100644 --- a/test/integration/components/testserver/duplicate_testserver.sh +++ b/test/integration/components/testserver/duplicate_testserver.sh @@ -4,7 +4,7 @@ run_testserver() { # prefix for start ports. E.g. 808 sp=$1 - STD_PORT=${1}0 GIN_PORT=${1}1 GORILLA_PORT=${1}2 GORILLA_MID_PORT=${1}3 GRPC_PORT=${1}4 GRPC_TLS_PORT=${1}5 GORILLA_MID2_PORT=${1}7 STD_TLS_PORT=${1}8 ./duped_service + STD_PORT=${1}0 GIN_PORT=${1}1 GORILLA_PORT=${1}2 GORILLA_MID_PORT=${1}3 GRPC_PORT=${1}4 GRPC_TLS_PORT=${1}5 GORILLA_MID2_PORT=${1}7 STD_TLS_PORT=${1}8 JSRONRPC_PORT=${1}9 ./duped_service } # runs testserver twice From be607313c27e76a1b003d64cc5ff038aa2754c08 Mon Sep 17 00:00:00 2001 From: titaneric Date: Wed, 25 Jun 2025 18:39:36 +0000 Subject: [PATCH 31/40] attempt nested trace test for jsonrpc (WIP) --- .../components/testserver/jsonrpc/jsonrpc.go | 16 ++++++++++++ .../components/testserver/std/std.go | 5 ++-- .../docker-compose-multiexec-host.yml | 1 + test/integration/multiprocess_test.go | 6 ++--- test/integration/traces_test.go | 25 +++++++++++++++++-- 5 files changed, 46 insertions(+), 7 deletions(-) diff --git a/test/integration/components/testserver/jsonrpc/jsonrpc.go b/test/integration/components/testserver/jsonrpc/jsonrpc.go index 68fb3a53b7..ab077813ad 100644 --- a/test/integration/components/testserver/jsonrpc/jsonrpc.go +++ b/test/integration/components/testserver/jsonrpc/jsonrpc.go @@ -23,6 +23,22 @@ func (t *Arith) Multiply(args *Args, reply *int) error { return nil } +func (t *Arith) Traceme(args *Args, reply *int) error { + requestURL := "http://pytestserver:8083/tracemetoo" + + slog.Debug("calling", "url", requestURL) + + res, err := http.Get(requestURL) + if err != nil { + slog.Error("error making http request", "error", err) + return err + } + + defer res.Body.Close() + t.Multiply(args, reply) + return nil +} + // ReadWriteCloserWrapper wraps an io.Reader and io.Writer to implement io.ReadWriteCloser. type ReadWriteCloserWrapper struct { io.Reader diff --git a/test/integration/components/testserver/std/std.go b/test/integration/components/testserver/std/std.go index 04a6d7e47a..a0ae83a3c8 100644 --- a/test/integration/components/testserver/std/std.go +++ b/test/integration/components/testserver/std/std.go @@ -1,6 +1,7 @@ package std import ( + "bytes" "context" "crypto/tls" "fmt" @@ -140,11 +141,11 @@ func echoLowPort(rw http.ResponseWriter) { } func echoDist(rw http.ResponseWriter) { - requestURL := "http://pytestserver:8083/tracemetoo" + requestURL := "http://testserver:8088/jsonrpc" slog.Debug("calling", "url", requestURL) - res, err := http.Get(requestURL) + res, err := http.Post(requestURL, "application/json", bytes.NewReader([]byte(`{"jsonrpc":"2.0","method":"Arith.Traceme","params":{"A":1,"B":2},"id":1}`))) if err != nil { slog.Error("error making http request", "error", err) rw.WriteHeader(http.StatusInternalServerError) diff --git a/test/integration/docker-compose-multiexec-host.yml b/test/integration/docker-compose-multiexec-host.yml index b544402d12..4af48a1a14 100644 --- a/test/integration/docker-compose-multiexec-host.yml +++ b/test/integration/docker-compose-multiexec-host.yml @@ -8,6 +8,7 @@ services: image: hatest-testserver ports: - "8080:8080" + - "8088:8088" environment: LOG_LEVEL: DEBUG depends_on: diff --git a/test/integration/multiprocess_test.go b/test/integration/multiprocess_test.go index f04bd03053..2569c8636e 100644 --- a/test/integration/multiprocess_test.go +++ b/test/integration/multiprocess_test.go @@ -96,7 +96,7 @@ func TestMultiProcess(t *testing.T) { }) if kprobeTracesEnabled() { - t.Run("Nested traces with kprobes: rust -> java -> node -> go -> python -> rails", func(t *testing.T) { + t.Run("Nested traces with kprobes: rust -> java -> node -> go -> go jsonrpc -> python -> rails", func(t *testing.T) { testNestedHTTPTracesKProbes(t) }) @@ -120,7 +120,7 @@ func TestMultiProcessAppCP(t *testing.T) { require.NoError(t, compose.Up()) if kprobeTracesEnabled() { - t.Run("Nested traces with kprobes: rust -> java -> node -> go -> python -> rails", func(t *testing.T) { + t.Run("Nested traces with kprobes: rust -> java -> node -> go -> go jsonrpc -> python -> rails", func(t *testing.T) { testNestedHTTPTracesKProbes(t) }) } @@ -136,7 +136,7 @@ func TestMultiProcessAppCPNoIP(t *testing.T) { require.NoError(t, compose.Up()) if kprobeTracesEnabled() { - t.Run("Nested traces with kprobes: rust -> java -> node -> go -> python -> rails", func(t *testing.T) { + t.Run("Nested traces with kprobes: rust -> java -> node -> go -> go jsonrpc -> python -> rails", func(t *testing.T) { testNestedHTTPTracesKProbes(t) }) } diff --git a/test/integration/traces_test.go b/test/integration/traces_test.go index 5a164d6e4b..4b47fab42b 100644 --- a/test/integration/traces_test.go +++ b/test/integration/traces_test.go @@ -656,6 +656,7 @@ func testNestedHTTPTracesKProbes(t *testing.T) { waitForRubyTestComponents(t, "http://localhost:3041") // ruby waitForTestComponentsSub(t, "http://localhost:8086", "/greeting") // java waitForTestComponents(t, "http://localhost:8091") // rust + waitForTestComponents(t, instrumentedServiceJSONRPCURL) // go jsonrpc // Add and check for specific trace ID // Run couple of requests to make sure we flush out any transactions that might be @@ -664,8 +665,8 @@ func testNestedHTTPTracesKProbes(t *testing.T) { doHTTPGet(t, "http://localhost:8091/dist", 200) } - // rust -> java -> nodejs -> go -> python -> rails - // /dist2 -> /jtrace2 -> /traceme -> /gotracemetoo -> /tracemetoo -> /users + // rust -> java -> nodejs -> go -> go jsonrpc -> python -> rails + // /dist2 -> /jtrace2 -> /traceme -> /gotracemetoo -> /jsonrpc -> /tracemetoo -> /users // Get the first 5 traces var multipleTraces []jaeger.Trace @@ -765,6 +766,26 @@ func testNestedHTTPTracesKProbes(t *testing.T) { ) assert.Empty(t, sd, sd.String()) + // Check the information of the go jsonrpc parent span + res = trace.FindByOperationName("Arith.T /jsonrpc", "server") + require.Len(t, res, 1) + parent = res[0] + require.NotEmpty(t, parent.TraceID) + require.Equal(t, traceID, parent.TraceID) + require.NotEmpty(t, parent.SpanID) + // check duration is at least 2us + assert.Less(t, (2 * time.Microsecond).Microseconds(), parent.Duration) + // check span attributes + sd = parent.Diff( + jaeger.Tag{Key: "http.request.method", Type: "string", Value: "Arith.T"}, + jaeger.Tag{Key: "http.response.status_code", Type: "int64", Value: float64(200)}, + jaeger.Tag{Key: "url.path", Type: "string", Value: "/jsonrpc"}, + jaeger.Tag{Key: "server.port", Type: "int64", Value: float64(8088)}, + jaeger.Tag{Key: "http.route", Type: "string", Value: "/jsonrpc"}, + jaeger.Tag{Key: "span.kind", Type: "string", Value: "server"}, + ) + assert.Empty(t, sd, sd.String()) + // Check the information of the python parent span res = trace.FindByOperationName("GET /tracemetoo", "server") require.Len(t, res, 1) From 01e12d54f82663cf89c07691fb8b4cfde502ef5c Mon Sep 17 00:00:00 2001 From: titaneric Date: Thu, 26 Jun 2025 16:51:49 +0000 Subject: [PATCH 32/40] fix jsonrpc payload and update jsonrpc logger --- .../components/testserver/jsonrpc/jsonrpc.go | 17 +++++++++++------ .../components/testserver/std/std.go | 2 +- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/test/integration/components/testserver/jsonrpc/jsonrpc.go b/test/integration/components/testserver/jsonrpc/jsonrpc.go index ab077813ad..fe9e6cf6d1 100644 --- a/test/integration/components/testserver/jsonrpc/jsonrpc.go +++ b/test/integration/components/testserver/jsonrpc/jsonrpc.go @@ -15,18 +15,22 @@ type Args struct { } // Arith provides methods for arithmetic operations. -type Arith struct{} +type Arith struct { + Logger *slog.Logger +} // Multiply multiplies two numbers and returns the result. func (t *Arith) Multiply(args *Args, reply *int) error { + t.Logger.Debug("calling", "method", "Arith.Multiply") *reply = args.A * args.B return nil } func (t *Arith) Traceme(args *Args, reply *int) error { + t.Logger.Debug("calling", "method", "Arith.Traceme") requestURL := "http://pytestserver:8083/tracemetoo" - slog.Debug("calling", "url", requestURL) + t.Logger.Debug("calling", "url", requestURL) res, err := http.Get(requestURL) if err != nil { @@ -35,8 +39,7 @@ func (t *Arith) Traceme(args *Args, reply *int) error { } defer res.Body.Close() - t.Multiply(args, reply) - return nil + return t.Multiply(args, reply) } // ReadWriteCloserWrapper wraps an io.Reader and io.Writer to implement io.ReadWriteCloser. @@ -51,10 +54,12 @@ func (w *ReadWriteCloserWrapper) Close() error { } func Setup(port int) { - arith := new(Arith) + log := slog.With("component", "jsonrpc.Arith") + arith := &Arith{ + Logger: log, + } _ = rpc.Register(arith) - log := slog.With("component", "jsonrpc.Server") address := fmt.Sprintf(":%d", port) log.Info("starting JSON-RPC server", "address", address) err := http.ListenAndServe(address, HTTPHandler(log)) diff --git a/test/integration/components/testserver/std/std.go b/test/integration/components/testserver/std/std.go index a0ae83a3c8..79bf27df4c 100644 --- a/test/integration/components/testserver/std/std.go +++ b/test/integration/components/testserver/std/std.go @@ -145,7 +145,7 @@ func echoDist(rw http.ResponseWriter) { slog.Debug("calling", "url", requestURL) - res, err := http.Post(requestURL, "application/json", bytes.NewReader([]byte(`{"jsonrpc":"2.0","method":"Arith.Traceme","params":{"A":1,"B":2},"id":1}`))) + res, err := http.Post(requestURL, "application/json", bytes.NewReader([]byte(`{"jsonrpc":"2.0","method":"Arith.Traceme","params":[{"A":1,"B":2}],"id":1}`))) if err != nil { slog.Error("error making http request", "error", err) rw.WriteHeader(http.StatusInternalServerError) From 286df52adf62d469b4ab17d64107c30f5045c800 Mon Sep 17 00:00:00 2001 From: titaneric Date: Thu, 26 Jun 2025 17:39:27 +0000 Subject: [PATCH 33/40] add k8s integration test for jsonrpc server --- .../k8s_daemonset_multi_node_traces_test.go | 12 ++++++++++-- .../k8s/manifests/05-uninstrumented-few-services.yml | 6 ++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/test/integration/k8s/daemonset_multi_node/k8s_daemonset_multi_node_traces_test.go b/test/integration/k8s/daemonset_multi_node/k8s_daemonset_multi_node_traces_test.go index 4eee6cfe5e..837ab92968 100644 --- a/test/integration/k8s/daemonset_multi_node/k8s_daemonset_multi_node_traces_test.go +++ b/test/integration/k8s/daemonset_multi_node/k8s_daemonset_multi_node_traces_test.go @@ -20,19 +20,20 @@ import ( // For the this scenario we run two worker nodes, with the following structure: // - worker 1: // testserver [go app] port: 8080 +// testserver [go jsonrpc app] port: 8088 // - worker 2: // pythonserver [python app] port: 8083 // ruby on rails [ruby app] port: 3040 // // The call flow is as follows: // -// testserver [/gotracemetoo] -> Python server [/tracemetoo] -> Ruby server [/users] +// testserver [/gotracemetoo] -> go jsonrpc [/jsonrpc] -> Python server [/tracemetoo] -> Ruby server [/users] // // They should all have the same traceID. Across nodes the TCP context propagation (OTEL_EBPF_BPF_CONTEXT_PROPAGATION) // connects the dots, while on the same node, the networking is optimized and we rely on black-box context propagation to // connect the services. func TestMultiNodeTracing(t *testing.T) { - feat := features.New("Beyla is able to generate distributed traces go->python->ruby"). + feat := features.New("Beyla is able to generate distributed traces go->go jsonrpc->python->ruby"). Assess("it sends connected traces for all services", func(ctx context.Context, t *testing.T, _ *envconf.Config) context.Context { var trace jaeger.Trace @@ -68,6 +69,13 @@ func TestMultiNodeTracing(t *testing.T) { }, trace.Processes[parent.ProcessID].Tags) require.Empty(t, sd, sd.String()) + // Check the information of the Go jsonrpc span + res = trace.FindByOperationName("Arith.T /jsonrpc", "server") + require.Len(t, res, 1) + parent = res[0] + require.NotEmpty(t, parent.TraceID) + + require.Equal(t, traceID, parent.TraceID) // Check the information of the Python span res = trace.FindByOperationName("GET /tracemetoo", "server") require.Len(t, res, 1) diff --git a/test/integration/k8s/manifests/05-uninstrumented-few-services.yml b/test/integration/k8s/manifests/05-uninstrumented-few-services.yml index ae9ffc4ba5..a81eca0139 100644 --- a/test/integration/k8s/manifests/05-uninstrumented-few-services.yml +++ b/test/integration/k8s/manifests/05-uninstrumented-few-services.yml @@ -10,6 +10,9 @@ spec: - port: 8080 name: http0 targetPort: http0 + - port: 8088 + name: http1 + targetPort: http1 --- apiVersion: v1 kind: Service @@ -112,6 +115,9 @@ spec: - containerPort: 8080 hostPort: 8080 name: http0 + - containerPort: 8088 + hostPort: 8088 + name: http1 env: - name: LOG_LEVEL value: "DEBUG" From 06d93ac3915d5897b5da1ac0fa3352ea2a13a013 Mon Sep 17 00:00:00 2001 From: titaneric Date: Thu, 26 Jun 2025 17:42:25 +0000 Subject: [PATCH 34/40] fix testcase format --- .../k8s_daemonset_multi_node_traces_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/k8s/daemonset_multi_node/k8s_daemonset_multi_node_traces_test.go b/test/integration/k8s/daemonset_multi_node/k8s_daemonset_multi_node_traces_test.go index 837ab92968..3309ade508 100644 --- a/test/integration/k8s/daemonset_multi_node/k8s_daemonset_multi_node_traces_test.go +++ b/test/integration/k8s/daemonset_multi_node/k8s_daemonset_multi_node_traces_test.go @@ -74,8 +74,8 @@ func TestMultiNodeTracing(t *testing.T) { require.Len(t, res, 1) parent = res[0] require.NotEmpty(t, parent.TraceID) - require.Equal(t, traceID, parent.TraceID) + // Check the information of the Python span res = trace.FindByOperationName("GET /tracemetoo", "server") require.Len(t, res, 1) From f69506f3e4ae315f903f49cf3e00ebcb01a3ce9d Mon Sep 17 00:00:00 2001 From: titaneric Date: Thu, 26 Jun 2025 18:39:43 +0000 Subject: [PATCH 35/40] Increase test timeout to 5 min for k8s integration test --- .../daemonset_multi_node/k8s_daemonset_multi_node_main_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/k8s/daemonset_multi_node/k8s_daemonset_multi_node_main_test.go b/test/integration/k8s/daemonset_multi_node/k8s_daemonset_multi_node_main_test.go index b95c6c8c10..2cf2b276b6 100644 --- a/test/integration/k8s/daemonset_multi_node/k8s_daemonset_multi_node_main_test.go +++ b/test/integration/k8s/daemonset_multi_node/k8s_daemonset_multi_node_main_test.go @@ -16,7 +16,7 @@ import ( ) const ( - testTimeout = 3 * time.Minute + testTimeout = 5 * time.Minute jaegerQueryURL = "http://localhost:36686/api/traces" ) From cfa2be11aec414529c208654338346ceb9b11cfc Mon Sep 17 00:00:00 2001 From: titaneric Date: Fri, 27 Jun 2025 16:58:36 +0000 Subject: [PATCH 36/40] restore timeout --- .../daemonset_multi_node/k8s_daemonset_multi_node_main_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/k8s/daemonset_multi_node/k8s_daemonset_multi_node_main_test.go b/test/integration/k8s/daemonset_multi_node/k8s_daemonset_multi_node_main_test.go index 2cf2b276b6..b95c6c8c10 100644 --- a/test/integration/k8s/daemonset_multi_node/k8s_daemonset_multi_node_main_test.go +++ b/test/integration/k8s/daemonset_multi_node/k8s_daemonset_multi_node_main_test.go @@ -16,7 +16,7 @@ import ( ) const ( - testTimeout = 5 * time.Minute + testTimeout = 3 * time.Minute jaegerQueryURL = "http://localhost:36686/api/traces" ) From f96fba91690aad8e7785a31d9effad7cab8d0e0d Mon Sep 17 00:00:00 2001 From: titaneric Date: Fri, 27 Jun 2025 18:33:07 +0000 Subject: [PATCH 37/40] Add delicated pattern for jsonrpc path in beyla's integration test --- .../integration/k8s/manifests/06-beyla-daemonset-multi-node.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/integration/k8s/manifests/06-beyla-daemonset-multi-node.yml b/test/integration/k8s/manifests/06-beyla-daemonset-multi-node.yml index 1130216237..dbfc0a1745 100644 --- a/test/integration/k8s/manifests/06-beyla-daemonset-multi-node.yml +++ b/test/integration/k8s/manifests/06-beyla-daemonset-multi-node.yml @@ -20,6 +20,8 @@ data: - exe_path: "/testserver" namespace: integration-test routes: + patterns: + - /jsonrpc unmatched: heuristic otel_metrics_export: endpoint: http://otelcol:4318 From 5a9be0123131511337de678c4254a7996ac9e2b0 Mon Sep 17 00:00:00 2001 From: titaneric Date: Fri, 4 Jul 2025 15:53:10 +0000 Subject: [PATCH 38/40] fix prometheus metrics name in integration test --- test/integration/red_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/red_test.go b/test/integration/red_test.go index d44f6f6688..60e2f4384a 100644 --- a/test/integration/red_test.go +++ b/test/integration/red_test.go @@ -247,7 +247,7 @@ func testSpanMetricsForJSONRPCHTTP(t *testing.T, svcName, svcNs string) { // Test span metrics test.Eventually(t, testTimeout, func(t require.TestingT) { var err error - results, err = pq.Query(`traces_span_metrics_duration_count{` + + results, err = pq.Query(`traces_span_metrics_duration_seconds_count{` + `span_kind="SPAN_KIND_SERVER",` + `status_code="STATUS_CODE_UNSET",` + // 404 is OK for server spans `service_namespace="` + svcNs + `",` + From 1bc2d88774205af3a319645958068e9f08b5737d Mon Sep 17 00:00:00 2001 From: titaneric Date: Fri, 4 Jul 2025 16:55:26 +0000 Subject: [PATCH 39/40] replace bpf_memicmp with stricmp --- bpf/gotracer/go_nethttp.c | 3 +-- bpf/gotracer/protocol_jsonrpc.h | 3 ++- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bpf/gotracer/go_nethttp.c b/bpf/gotracer/go_nethttp.c index e4abcf096a..838efff58e 100644 --- a/bpf/gotracer/go_nethttp.c +++ b/bpf/gotracer/go_nethttp.c @@ -345,8 +345,7 @@ static __always_inline void handle_content_type_header(server_http_func_invocati // Matches the header in the buffer and returns a pointer to the value part of the header. static __always_inline unsigned char *match_header( const unsigned char *buf, u32 safe_len, const char *header, u32 header_len, u32 value_len) { - if (safe_len >= header_len + value_len && - bpf_memicmp((const char *)buf, header, header_len) == 0) { + if (safe_len >= header_len + value_len && stricmp((const char *)buf, header, header_len) == 0) { return (unsigned char *)(buf + header_len); } return NULL; diff --git a/bpf/gotracer/protocol_jsonrpc.h b/bpf/gotracer/protocol_jsonrpc.h index 0cca29b0d9..8f21beb97f 100644 --- a/bpf/gotracer/protocol_jsonrpc.h +++ b/bpf/gotracer/protocol_jsonrpc.h @@ -3,6 +3,7 @@ #include #include #include +#include #include static const char k_jsonrpc_key[] = "\"jsonrpc\""; @@ -61,7 +62,7 @@ static __always_inline u32 json_str_value(const unsigned char *body, // Returns 1 if equal, 0 otherwise. static __always_inline u8 json_value_eq(const char *start, const char *val, u32 val_len) { - return bpf_memicmp(start, val, val_len) == 0; + return stricmp(start, val, val_len) == 0; } // Extracts a JSON string value starting at a given position. From cce786c7925a072a45e81ad14b60a6f3cf0ea2ef Mon Sep 17 00:00:00 2001 From: titaneric Date: Sat, 5 Jul 2025 15:30:07 +0000 Subject: [PATCH 40/40] update type handling for stricmp --- bpf/gotracer/go_nethttp.c | 2 +- bpf/gotracer/protocol_jsonrpc.h | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/bpf/gotracer/go_nethttp.c b/bpf/gotracer/go_nethttp.c index 838efff58e..aa3c4e5647 100644 --- a/bpf/gotracer/go_nethttp.c +++ b/bpf/gotracer/go_nethttp.c @@ -345,7 +345,7 @@ static __always_inline void handle_content_type_header(server_http_func_invocati // Matches the header in the buffer and returns a pointer to the value part of the header. static __always_inline unsigned char *match_header( const unsigned char *buf, u32 safe_len, const char *header, u32 header_len, u32 value_len) { - if (safe_len >= header_len + value_len && stricmp((const char *)buf, header, header_len) == 0) { + if (safe_len >= header_len + value_len && stricmp((const char *)buf, header, header_len)) { return (unsigned char *)(buf + header_len); } return NULL; diff --git a/bpf/gotracer/protocol_jsonrpc.h b/bpf/gotracer/protocol_jsonrpc.h index 8f21beb97f..9a8537a882 100644 --- a/bpf/gotracer/protocol_jsonrpc.h +++ b/bpf/gotracer/protocol_jsonrpc.h @@ -59,10 +59,9 @@ static __always_inline u32 json_str_value(const unsigned char *body, } // Compares a JSON value at start with a given value. -// Returns 1 if equal, 0 otherwise. -static __always_inline u8 json_value_eq(const char *start, const char *val, u32 val_len) { +static __always_inline bool json_value_eq(const char *start, const char *val, u32 val_len) { - return stricmp(start, val, val_len) == 0; + return stricmp(start, val, val_len); } // Extracts a JSON string value starting at a given position.