Use separate execIDs for clone and subsequent exec events

In ed2dc24
we introduced the clone events and subsequent exec events to have the
same execID. An example here is:
  clone()   id=1
    exec()  id=1
    exec()  id=1
  exit()    id=1

This is done in order to have the same execID in exit (and be able to
cleanup the process cache in the uswr space).

On the other hand, this approach has an issue. Let's assume the
following scenario:
  clone()    id=1
    kprobe() id=1
    exec()   id=1
    kprobe() id=1
    exec()   id=1
  exit()    id=1

If a kprobe event comes OOO we cannot know when this is actually happen
(and use the correct process information). In that case if the first
kprobe event comes after the first exec we should add this to the
eventcache in order to wait for the process info.

The previous commit reverts ed2dc24
and this commit enables clone and subsequent exec events to all
have a separate (and unique) execID. In order to fix the cleanup issue
we send the pid and ktime of the process needed cleanup with the exec
event.

Using the new approach an example is:
  clone()             id=1
    kprobe()          id=1
    exec()            id=2 [will also cleanup id=1]
    kprobe()          id=2
    exec()            id=3 [will also cleanup id=2]
  exit()              id=3

Signed-off-by: Anastasios Papagiannis <[email protected]>

bpf/lib/process.h

@@ -256,6 +256,7 @@ struct msg_execve_event {
 	__u64 parent_flags;
 	struct msg_capabilities caps;
 	struct msg_ns ns;
+	struct msg_execve_key cleanup_key;
 	/* if add anything above please also update the args of
 	 * validate_msg_execve_size() in bpf_execve_event.c */
 	union {

bpf/process/bpf_execve_event.c

@@ -221,6 +221,7 @@ execve_send(struct sched_execve_args *ctx)
 
 	curr = execve_map_get(pid);
 	if (curr) {
+		event->cleanup_key = curr->key;
 #if defined(__NS_CHANGES_FILTER) || defined(__CAP_CHANGES_FILTER)
 		/* if this exec event preceds a clone, initialize  capabilities
 		 * and namespaces as well.
@@ -256,7 +257,7 @@ execve_send(struct sched_execve_args *ctx)
 		sizeof(struct msg_common) + sizeof(struct msg_k8s) +
 		sizeof(struct msg_execve_key) + sizeof(__u64) +
 		sizeof(struct msg_capabilities) + sizeof(struct msg_ns) +
-		execve->size);
+		sizeof(struct msg_execve_key) + execve->size);
 	perf_event_output(ctx, &tcpmon_map, BPF_F_CURRENT_CPU, event, size);
 	return 0;
 }

pkg/api/processapi/processapi.go

@@ -67,22 +67,24 @@ type MsgK8sUnix struct {
 }
 
 type MsgExecveEvent struct {
-	Common       MsgCommon
-	Kube         MsgK8s
-	Parent       MsgExecveKey
-	ParentFlags  uint64
-	Capabilities MsgCapabilities
-	Namespaces   MsgNamespaces
+	Common         MsgCommon
+	Kube           MsgK8s
+	Parent         MsgExecveKey
+	ParentFlags    uint64
+	Capabilities   MsgCapabilities
+	Namespaces     MsgNamespaces
+	CleanupProcess MsgExecveKey
 }
 
 type MsgExecveEventUnix struct {
-	Common       MsgCommon
-	Kube         MsgK8sUnix
-	Parent       MsgExecveKey
-	ParentFlags  uint64
-	Capabilities MsgCapabilities
-	Namespaces   MsgNamespaces
-	Process      MsgProcess
+	Common         MsgCommon
+	Kube           MsgK8sUnix
+	Parent         MsgExecveKey
+	ParentFlags    uint64
+	Capabilities   MsgCapabilities
+	Namespaces     MsgNamespaces
+	CleanupProcess MsgExecveKey
+	Process        MsgProcess
 }
 
 type MsgCloneEvent struct {

pkg/grpc/exec/exec.go

@@ -4,7 +4,6 @@ package exec
 
 import (
 	"fmt"
-	"strings"
 
 	"github.com/cilium/tetragon/api/v1/tetragon"
 	"github.com/cilium/tetragon/pkg/api/ops"
@@ -31,6 +30,16 @@ const (
 	ProcessRefCnt = 1
 )
 
+func execCleanup(msg *MsgExecveEventUnix) {
+	if msg.CleanupProcess.Ktime != 0 {
+		ev := &MsgProcessCleanupEventUnix{
+			PID:   msg.CleanupProcess.Pid,
+			Ktime: msg.CleanupProcess.Ktime,
+		}
+		ev.HandleMessage()
+	}
+}
+
 // GetProcessExec returns Exec protobuf message for a given process, including the ancestor list.
 func GetProcessExec(event *MsgExecveEventUnix, useCache bool) *tetragon.ProcessExec {
 	var tetragonParent *tetragon.Process
@@ -51,15 +60,6 @@ func GetProcessExec(event *MsgExecveEventUnix, useCache bool) *tetragon.ProcessE
 		logger.GetLogger().WithError(err).WithField("processId", processId).WithField("parentId", parentId).Debugf("Failed to annotate process with capabilities and namespaces info")
 	}
 
-	// If this is not a clone we need to decrement parent refcnt because
-	// the parent has been replaced and will not get its own exit event.
-	// The new process will hold needed refcnts until it is destroyed.
-	if strings.Contains(tetragonProcess.Flags, "clone") == false &&
-		strings.Contains(tetragonProcess.Flags, "procFS") == false &&
-		parent != nil {
-		parent.RefDec()
-	}
-
 	tetragonEvent := &tetragon.ProcessExec{
 		Process: tetragonProcess,
 		Parent:  tetragonParent,
@@ -78,6 +78,9 @@ func GetProcessExec(event *MsgExecveEventUnix, useCache bool) *tetragon.ProcessE
 		tetragonEvent.Parent = parent.GetProcessCopy()
 	}
 
+	// do we need to cleanup anything?
+	execCleanup(event)
+
 	return tetragonEvent
 }
 
@@ -127,11 +130,13 @@ func (msg *MsgExecveEventUnix) Retry(internal *process.ProcessInternal, ev notif
 		if parent == nil {
 			return err
 		}
-		if strings.Contains(proc.Flags, "clone") == true {
-			parent.RefInc()
-		}
+		parent.RefInc()
+		ev.SetParent(parent.GetProcessCopy())
 	}
 
+	// do we need to cleanup anything?
+	execCleanup(msg)
+
 	return nil
 }
 
@@ -306,3 +311,62 @@ func (msg *MsgExitEventUnix) Cast(o interface{}) notify.Message {
 	t := o.(tetragonAPI.MsgExitEvent)
 	return &MsgExitEventUnix{MsgExitEvent: t}
 }
+
+type MsgProcessCleanupEventUnix struct {
+	PID        uint32
+	Ktime      uint64
+	RefCntDone [2]bool
+}
+
+func (msg *MsgProcessCleanupEventUnix) Notify() bool {
+	return false
+}
+
+func (msg *MsgProcessCleanupEventUnix) RetryInternal(ev notify.Event, timestamp uint64) (*process.ProcessInternal, error) {
+	internal, parent := process.GetParentProcessInternal(msg.PID, timestamp)
+	var err error
+
+	if parent != nil {
+		if !msg.RefCntDone[ParentRefCnt] {
+			parent.RefDec()
+			msg.RefCntDone[ParentRefCnt] = true
+		}
+	} else {
+		err = eventcache.ErrFailedToGetParentInfo
+	}
+
+	if internal != nil {
+		if !msg.RefCntDone[ProcessRefCnt] {
+			internal.RefDec()
+			msg.RefCntDone[ProcessRefCnt] = true
+		}
+	} else {
+		err = eventcache.ErrFailedToGetProcessInfo
+	}
+
+	if err == nil {
+		return internal, err
+	}
+	return nil, err
+}
+
+func (msg *MsgProcessCleanupEventUnix) Retry(internal *process.ProcessInternal, ev notify.Event) error {
+	return nil
+}
+
+func (msg *MsgProcessCleanupEventUnix) HandleMessage() *tetragon.GetEventsResponse {
+	msg.RefCntDone = [2]bool{false, false}
+	if process, parent := process.GetParentProcessInternal(msg.PID, msg.Ktime); process != nil && parent != nil {
+		parent.RefDec()
+		process.RefDec()
+	} else {
+		if ec := eventcache.Get(); ec != nil {
+			ec.Add(nil, nil, msg.Ktime, msg)
+		}
+	}
+	return nil
+}
+
+func (msg *MsgProcessCleanupEventUnix) Cast(o interface{}) notify.Message {
+	return &MsgProcessCleanupEventUnix{}
+}

pkg/sensors/exec/exec.go

@@ -50,6 +50,8 @@ func msgToExecveUnix(m *processapi.MsgExecveEvent) *exec.MsgExecveEventUnix {
 	unix.Namespaces.CgroupInum = m.Namespaces.CgroupInum
 	unix.Namespaces.UserInum = m.Namespaces.UserInum
 
+	unix.CleanupProcess = m.CleanupProcess
+
 	return unix
 }