[system-probe][NET-2891] Fix tcp retransmit count (#14740)

* [NET-2891] initial pass at changes to prebuilt code

* [NET-2891] use retrans_out for runtime compiled tcp_retransmit counter

* [NET-2891] runtime compiled version of tcp_retrans updates

* [NET-2891] remove debug comment

* [NET-2891] fix log

* [NET-2891] update bytecode

* [NET-2891] code review comments, regenerate license

* [NET-2891] newline

* [NET-2891] fix probe definitions

* [NET-2891] update comment

* [NET-2891] runtime compilation fixes

* [NET-2891] fix byte padding for args init

* [NET-2891] fix formatting

* testing debug logic

* more debug logic, added some config for the map

* [NET-2891] enable kretprobe and remove debug

* [NET-2891] disable bpf debug be default

* [NET-2891] update bytecode

* [NET-2891] make function as maybe unused

* [NET-2891] handle different paths of incremental vs absolute retransmit counters

* [NET-2891] use enum to track increment vs absolute retransmits

* [NET-2891] change enum values

* [NET-2891] move retrans code to runtime tracer

* pulled in new gitignore

* Revert "pulled in new gitignore"

This reverts commit b4b0df5.

* remove runtime gen files, code review comments

* [NET-2891] use retransmit count none in runtime tracer

* [NET-2891] use retransmit_count_none in handle_tcp_stats

* [NET-2891] nit comments from code review

* [NET-2891] try to get runtime compilation working on 4.4 kernel

pkg/network/ebpf/c/prebuilt/tracer.c

@@ -441,16 +441,43 @@ int kprobe__tcp_retransmit_skb(struct pt_regs *ctx) {
     struct sock *sk = (struct sock *)PT_REGS_PARM1(ctx);
     int segs = (int)PT_REGS_PARM3(ctx);
     log_debug("kprobe/tcp_retransmit: segs: %d\n", segs);
-
-    return handle_retransmit(sk, segs);
+    u64 pid_tgid = bpf_get_current_pid_tgid();
+    tcp_retransmit_skb_args_t args = {};
+    args.sk = sk;
+    args.segs = segs;
+    bpf_map_update_with_telemetry(pending_tcp_retransmit_skb, &pid_tgid, &args, BPF_ANY);
+    return 0;
 }
 
 SEC("kprobe/tcp_retransmit_skb/pre_4_7_0")
 int kprobe__tcp_retransmit_skb_pre_4_7_0(struct pt_regs *ctx) {
     struct sock *sk = (struct sock *)PT_REGS_PARM1(ctx);
     log_debug("kprobe/tcp_retransmit/pre_4_7_0\n");
+    u64 pid_tgid = bpf_get_current_pid_tgid();
+    tcp_retransmit_skb_args_t args = {};
+    args.sk = sk;
+    args.segs = 1;
+    bpf_map_update_with_telemetry(pending_tcp_retransmit_skb, &pid_tgid, &args, BPF_ANY);
+    return 0;
+}
 
-    return handle_retransmit(sk, 1);
+SEC("kretprobe/tcp_retransmit_skb")
+int kretprobe__tcp_retransmit_skb(struct pt_regs *ctx) {
+    int ret = PT_REGS_RC(ctx);
+    __u64 tid = bpf_get_current_pid_tgid();
+    if (ret < 0) {
+        bpf_map_delete_elem(&pending_tcp_retransmit_skb, &tid);
+        return 0;
+    }
+    tcp_retransmit_skb_args_t *args = bpf_map_lookup_elem(&pending_tcp_retransmit_skb, &tid);
+    if (args == NULL) {
+        return 0;
+    }
+    struct sock *sk = args->sk;
+    int segs = args->segs;
+    bpf_map_delete_elem(&pending_tcp_retransmit_skb, &tid);
+    log_debug("kretprobe/tcp_retransmit: segs: %d\n", segs);
+    return handle_retransmit(sk, segs, RETRANSMIT_COUNT_INCREMENT);
 }
 
 SEC("kprobe/tcp_set_state")
@@ -470,7 +497,7 @@ int kprobe__tcp_set_state(struct pt_regs *ctx) {
     }
 
     tcp_stats_t stats = { .state_transitions = (1 << state) };
-    update_tcp_stats(&t, stats);
+    update_tcp_stats(&t, stats, RETRANSMIT_COUNT_NONE);
 
     return 0;
 }

pkg/network/ebpf/c/runtime/tracer.c

@@ -397,16 +397,36 @@ int kprobe__skb_consume_udp(struct pt_regs *ctx) {
 
 SEC("kprobe/tcp_retransmit_skb")
 int kprobe__tcp_retransmit_skb(struct pt_regs *ctx) {
+    log_debug("kprobe/tcp_retransmit\n");
     struct sock *sk = (struct sock *)PT_REGS_PARM1(ctx);
+    u64 tid = bpf_get_current_pid_tgid();
+    tcp_retransmit_skb_args_t args = {};
+    args.sk = sk;
+    args.segs = 0;
+    bpf_map_update_with_telemetry(pending_tcp_retransmit_skb, &tid, &args, BPF_ANY);
 
-#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 7, 0)
-    int segs = 1;
-#else
-    int segs = (int)PT_REGS_PARM3(ctx);
-#endif
-    log_debug("kprobe/tcp_retransmit\n");
+    return 0;
+}
+
+SEC("kretprobe/tcp_retransmit_skb")
+int kretprobe__tcp_retransmit_skb(struct pt_regs *ctx) {
+    log_debug("kretprobe/tcp_retransmit\n");
+    u64 tid = bpf_get_current_pid_tgid();
+    if (PT_REGS_RC(ctx) < 0) {
+        bpf_map_delete_elem(&pending_tcp_retransmit_skb, &tid);
+        return 0;
+    }
+    tcp_retransmit_skb_args_t *args = bpf_map_lookup_elem(&pending_tcp_retransmit_skb, &tid);
+    if (args == NULL) {
+        return 0;
+    }
+    struct sock* sk = args->sk;
+    u32 retrans_out = 0;
+    bpf_probe_read_kernel_with_telemetry(&retrans_out, sizeof(retrans_out), &(tcp_sk(sk)->retrans_out));
 
-    return handle_retransmit(sk, segs);
+    bpf_map_delete_elem(&pending_tcp_retransmit_skb, &tid);
+
+    return handle_retransmit(sk, retrans_out, RETRANSMIT_COUNT_ABSOLUTE);
 }
 
 SEC("kprobe/tcp_set_state")
@@ -426,7 +446,7 @@ int kprobe__tcp_set_state(struct pt_regs *ctx) {
     }
 
     tcp_stats_t stats = { .state_transitions = (1 << state) };
-    update_tcp_stats(&t, stats);
+    update_tcp_stats(&t, stats, RETRANSMIT_COUNT_NONE);
 
     return 0;
 }

pkg/network/ebpf/c/tracer-maps.h

@@ -71,10 +71,17 @@ BPF_HASH_MAP(udp_port_bindings, port_binding_t, __u32, 0)
 /* Similar to pending_sockets this is used for capturing state between the call and return of the bind() system call.
  *
  * Keys: the PID returned by bpf_get_current_pid_tgid()
- * Values: the args of the bind call  being instrumented.
+ * Values: the args of the bind call being instrumented.
  */
 BPF_HASH_MAP(pending_bind, __u64, bind_syscall_args_t, 8192)
 
+/* Similar to pending_sockets this is used for capturing state between the call and return of the tcp_retransmit_skb() system call.
+ *
+ * Keys: the PID returned by bpf_get_current_pid_tgid()
+ * Values: the args of the tcp_retransmit_skb call being instrumented.
+ */
+BPF_HASH_MAP(pending_tcp_retransmit_skb, __u64, tcp_retransmit_skb_args_t, 8192)
+
 /* This map is used for telemetry in kernelspace
  * only key 0 is used
  * value is a telemetry object

pkg/network/ebpf/c/tracer-stats.h

@@ -5,6 +5,7 @@
 #include "tracer.h"
 #include "tracer-maps.h"
 #include "tracer-telemetry.h"
+#include <net/tcp.h>
 #include "sock-defines.h"
 #include "cookie.h"
 
@@ -134,7 +135,7 @@ static __always_inline void update_conn_stats(conn_tuple_t *t, size_t sent_bytes
     }
 }
 
-static __always_inline void update_tcp_stats(conn_tuple_t *t, tcp_stats_t stats) {
+static __always_inline void update_tcp_stats(conn_tuple_t *t, tcp_stats_t stats, retransmit_count_increment_t retrans_type) {
     // query stats without the PID from the tuple
     __u32 pid = t->pid;
     t->pid = 0;
@@ -149,7 +150,9 @@ static __always_inline void update_tcp_stats(conn_tuple_t *t, tcp_stats_t stats)
         return;
     }
 
-    if (stats.retransmits > 0) {
+    if (retrans_type == RETRANSMIT_COUNT_ABSOLUTE) {
+        val->retransmits = stats.retransmits;
+    } else if (retrans_type == RETRANSMIT_COUNT_INCREMENT && stats.retransmits > 0) {
         __sync_fetch_and_add(&val->retransmits, stats.retransmits);
     }
 
@@ -174,16 +177,16 @@ static __always_inline int handle_message(conn_tuple_t *t, size_t sent_bytes, si
     return 0;
 }
 
-static __always_inline int handle_retransmit(struct sock *sk, int segs) {
+static __always_inline int handle_retransmit(struct sock *sk, int count, retransmit_count_increment_t retrans_type) {
     conn_tuple_t t = {};
     u64 zero = 0;
 
     if (!read_conn_tuple(&t, sk, zero, CONN_TYPE_TCP)) {
         return 0;
     }
 
-    tcp_stats_t stats = { .retransmits = segs, .rtt = 0, .rtt_var = 0 };
-    update_tcp_stats(&t, stats);
+    tcp_stats_t stats = { .retransmits = count, .rtt = 0, .rtt_var = 0 };
+    update_tcp_stats(&t, stats, retrans_type);
 
     return 0;
 }
@@ -198,7 +201,7 @@ static __always_inline void handle_tcp_stats(conn_tuple_t* t, struct sock* sk, u
     if (state > 0) {
         stats.state_transitions = (1 << state);
     }
-    update_tcp_stats(t, stats);
+    update_tcp_stats(t, stats, RETRANSMIT_COUNT_NONE);
 }
 
 

pkg/network/ebpf/c/tracer.h

@@ -23,6 +23,13 @@ typedef enum
     PACKET_COUNT_INCREMENT = 2,
 } packet_count_increment_t;
 
+typedef enum
+{
+    RETRANSMIT_COUNT_NONE = 0,
+    RETRANSMIT_COUNT_ABSOLUTE = 1,
+    RETRANSMIT_COUNT_INCREMENT = 2,
+} retransmit_count_increment_t;
+
 #define CONN_DIRECTION_MASK 0b11
 
 typedef struct {
@@ -149,6 +156,11 @@ typedef struct {
     struct sock *sk;
 } bind_syscall_args_t;
 
+typedef struct {
+    struct sock *sk;
+    int segs;
+} tcp_retransmit_skb_args_t;
+
 typedef struct {
     __u32 netns;
     __u16 port;

pkg/network/ebpf/probes/probes.go

@@ -113,10 +113,12 @@ const (
 	// UDPDestroySockReturn traces the return of the udp_destroy_sock() system call
 	UDPDestroySockReturn ProbeName = "kretprobe/udp_destroy_sock"
 
-	// TCPRetransmit traces the return value for the tcp_retransmit_skb() system call
+	// TCPRetransmit traces the params for the tcp_retransmit_skb() system call
 	TCPRetransmit ProbeName = "kprobe/tcp_retransmit_skb"
-	// TCPRetransmitPre470 traces the return value for the tcp_retransmit_skb() system call on kernel version < 4.7
+	// TCPRetransmitPre470 traces the params for the tcp_retransmit_skb() system call on kernel version < 4.7
 	TCPRetransmitPre470 ProbeName = "kprobe/tcp_retransmit_skb/pre_4_7_0"
+	// TCPRetransmitRet traces the return value for the tcp_retransmit_skb() system call
+	TCPRetransmitRet ProbeName = "kretprobe/tcp_retransmit_skb"
 
 	// InetCskAcceptReturn traces the return value for the inet_csk_accept syscall
 	InetCskAcceptReturn ProbeName = "kretprobe/inet_csk_accept"

pkg/network/tracer/connection/kprobe/config.go

@@ -60,6 +60,7 @@ func enabledProbes(c *config.Config, runtimeTracer bool) (map[probes.ProbeName]s
 		enableProbe(enabled, probes.InetCskListenStop)
 		enableProbe(enabled, probes.TCPSetState)
 		enableProbe(enabled, selectVersionBasedProbe(runtimeTracer, kv, probes.TCPRetransmit, probes.TCPRetransmitPre470, kv470))
+		enableProbe(enabled, probes.TCPRetransmitRet)
 
 		missing, err := ebpf.VerifyKernelFuncs(ksymPath, []string{"sockfd_lookup_light"})
 		if err == nil && len(missing) == 0 {

pkg/network/tracer/connection/kprobe/manager.go

@@ -48,6 +48,7 @@ var mainProbes = map[probes.ProbeName]string{
 	probes.UDPv6RecvMsg:                   "kprobe__udpv6_recvmsg",
 	probes.UDPv6RecvMsgReturn:             "kretprobe__udpv6_recvmsg",
 	probes.TCPRetransmit:                  "kprobe__tcp_retransmit_skb",
+	probes.TCPRetransmitRet:               "kretprobe__tcp_retransmit_skb",
 	probes.InetCskAcceptReturn:            "kretprobe__inet_csk_accept",
 	probes.InetCskListenStop:              "kprobe__inet_csk_listen_stop",
 	probes.UDPDestroySock:                 "kprobe__udp_destroy_sock",