pkg/metrics: add event for tracing policy metrics

This patch adds metrics for tracing policy events. It uses the policy
name and a hook name as labels.

As an example, with this policy:
```
apiVersion: cilium.io/v1alpha1
kind: TracingPolicy
metadata:
  name: "kubectlexec-net"
spec:
  kprobes:
  - call: "tcp_connect"
    syscall: false
    args:
     - index: 0
       type: "sock"
    selectors:
    - matchPIDs:
      - operator: NotIn
        followForks: true
        isNamespacePID: true
        values:
        - 1
  - call: "tcp_close"
    syscall: false
    args:
     - index: 0
       type: "sock"
    selectors:
    - matchPIDs:
      - operator: NotIn
        followForks: true
        isNamespacePID: true
        values:
        - 1
  - call: "tcp_sendmsg"
    syscall: false
    args:
     - index: 0
       type: "sock"
     - index: 2
       type: int
    # follow any non-init pids (e.g., exec into container)
    selectors:
    - matchPIDs:
      - operator: NotIn
        followForks: true
        isNamespacePID: true
        values:
        - 1
```

We will see the following events:
```
tetragon_policy_stats{binary="/usr/bin/curl",hook="kprobe:tcp_close",namespace="default",pod="pizza",policy="kubectlexec-net"} 1                              │
tetragon_policy_stats{binary="/usr/bin/curl",hook="kprobe:tcp_connect",namespace="default",pod="pizza",policy="kubectlexec-net"} 1                            │
tetragon_policy_stats{binary="/usr/bin/curl",hook="kprobe:tcp_sendmsg",namespace="default",pod="pizza",policy="kubectlexec-net"} 1
```

Signed-off-by: Kornilios Kourtis <[email protected]>

Author: kkourt

pkg/grpc/tracing/tracing.go

@@ -3,6 +3,8 @@
 package tracing
 
 import (
+	"fmt"
+
 	"github.com/cilium/tetragon/api/v1/tetragon"
 	"github.com/cilium/tetragon/pkg/api/processapi"
 	"github.com/cilium/tetragon/pkg/api/tracingapi"
@@ -18,6 +20,7 @@ import (
 	"github.com/cilium/tetragon/pkg/reader/node"
 	"github.com/cilium/tetragon/pkg/reader/notify"
 	"github.com/cilium/tetragon/pkg/reader/path"
+	"github.com/cilium/tetragon/pkg/tracingpolicy"
 	"google.golang.org/protobuf/types/known/wrapperspb"
 )
 
@@ -224,6 +227,7 @@ type MsgGenericTracepointUnix struct {
 	Subsys     string
 	Event      string
 	Args       []tracingapi.MsgGenericTracepointArg
+	PolicyName string
 }
 
 func (msg *MsgGenericTracepointUnix) Notify() bool {
@@ -314,6 +318,13 @@ func (msg *MsgGenericTracepointUnix) Cast(o interface{}) notify.Message {
 	return &t
 }
 
+func (msg *MsgGenericTracepointUnix) PolicyInfo() tracingpolicy.PolicyInfo {
+	return tracingpolicy.PolicyInfo{
+		Name: msg.PolicyName,
+		Hook: fmt.Sprintf("tracepoint:%s/%s", msg.Subsys, msg.Event),
+	}
+}
+
 type MsgGenericKprobeUnix struct {
 	Common       processapi.MsgCommon
 	ProcessKey   processapi.MsgExecveKey
@@ -323,6 +334,7 @@ type MsgGenericKprobeUnix struct {
 	Action       uint64
 	FuncName     string
 	Args         []tracingapi.MsgGenericKprobeArg
+	PolicyName   string
 }
 
 func (msg *MsgGenericKprobeUnix) Notify() bool {
@@ -354,6 +366,13 @@ func (msg *MsgGenericKprobeUnix) Cast(o interface{}) notify.Message {
 	return &t
 }
 
+func (msg *MsgGenericKprobeUnix) PolicyInfo() tracingpolicy.PolicyInfo {
+	return tracingpolicy.PolicyInfo{
+		Name: msg.PolicyName,
+		Hook: fmt.Sprintf("kprobe:%s", msg.FuncName),
+	}
+}
+
 type MsgProcessLoaderUnix struct {
 	ProcessKey processapi.MsgExecveKey
 	Path       string
@@ -440,12 +459,20 @@ type MsgGenericUprobeUnix struct {
 	ProcessKey processapi.MsgExecveKey
 	Path       string
 	Symbol     string
+	PolicyName string
 }
 
 func (msg *MsgGenericUprobeUnix) Notify() bool {
 	return true
 }
 
+func (msg *MsgGenericUprobeUnix) PolicyInfo() tracingpolicy.PolicyInfo {
+	return tracingpolicy.PolicyInfo{
+		Name: msg.PolicyName,
+		Hook: fmt.Sprintf("uprobe:%s/%s", msg.Path, msg.Symbol),
+	}
+}
+
 func (msg *MsgGenericUprobeUnix) RetryInternal(ev notify.Event, timestamp uint64) (*process.ProcessInternal, error) {
 	return eventcache.HandleGenericInternal(ev, msg.ProcessKey.Pid, timestamp)
 }

pkg/metrics/eventmetrics/eventmetrics.go

@@ -14,6 +14,7 @@ import (
 	"github.com/cilium/tetragon/pkg/metrics/syscallmetrics"
 	v1 "github.com/cilium/tetragon/pkg/oldhubble/api/v1"
 	"github.com/cilium/tetragon/pkg/reader/exec"
+	"github.com/cilium/tetragon/pkg/tracingpolicy"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
 )
@@ -34,6 +35,12 @@ var (
 		Help:        "The total number of events dropped because listener buffer was full",
 		ConstLabels: nil,
 	}, nil)
+
+	policyStats = promauto.NewCounterVec(prometheus.CounterOpts{
+		Name:        consts.MetricNamePrefix + "policy_stats",
+		Help:        "Policy events calls observed.",
+		ConstLabels: nil,
+	}, []string{"policy", "hook", "namespace", "pod", "binary"})
 )
 
 func GetProcessInfo(process *tetragon.Process) (binary, pod, namespace string) {
@@ -60,7 +67,7 @@ func handleOriginalEvent(originalEvent interface{}) {
 	}
 }
 
-func handleProcessedEvent(processedEvent interface{}) {
+func handleProcessedEvent(pInfo *tracingpolicy.PolicyInfo, processedEvent interface{}) {
 	var eventType, namespace, pod, binary string
 	switch ev := processedEvent.(type) {
 	case *tetragon.GetEventsResponse:
@@ -75,10 +82,21 @@ func handleProcessedEvent(processedEvent interface{}) {
 		eventType = "unknown"
 	}
 	EventsProcessed.WithLabelValues(eventType, namespace, pod, binary).Inc()
+	if pInfo != nil && pInfo.Name != "" {
+		policyStats.
+			WithLabelValues(pInfo.Name, pInfo.Hook, namespace, pod, binary).
+			Inc()
+	}
 }
 
 func ProcessEvent(originalEvent interface{}, processedEvent interface{}) {
 	handleOriginalEvent(originalEvent)
-	handleProcessedEvent(processedEvent)
+
+	var policyInfo tracingpolicy.PolicyInfo
+	if policyEv, ok := originalEvent.(tracingpolicy.PolicyEvent); ok {
+		policyInfo = policyEv.PolicyInfo()
+	}
+
+	handleProcessedEvent(&policyInfo, processedEvent)
 	syscallmetrics.Handle(processedEvent)
 }

pkg/metrics/eventmetrics/eventmetrics_test.go

@@ -16,47 +16,47 @@ import (
 
 func TestHandleProcessedEvent(t *testing.T) {
 	assert.NoError(t, testutil.CollectAndCompare(EventsProcessed, strings.NewReader("")))
-	handleProcessedEvent(nil)
+	handleProcessedEvent(nil, nil)
 	// empty process
-	handleProcessedEvent(&tetragon.GetEventsResponse{Event: &tetragon.GetEventsResponse_ProcessKprobe{ProcessKprobe: &tetragon.ProcessKprobe{}}})
-	handleProcessedEvent(&tetragon.GetEventsResponse{Event: &tetragon.GetEventsResponse_ProcessExec{ProcessExec: &tetragon.ProcessExec{}}})
-	handleProcessedEvent(&tetragon.GetEventsResponse{Event: &tetragon.GetEventsResponse_ProcessTracepoint{ProcessTracepoint: &tetragon.ProcessTracepoint{}}})
-	handleProcessedEvent(&tetragon.GetEventsResponse{Event: &tetragon.GetEventsResponse_ProcessExit{ProcessExit: &tetragon.ProcessExit{}}})
+	handleProcessedEvent(nil, &tetragon.GetEventsResponse{Event: &tetragon.GetEventsResponse_ProcessKprobe{ProcessKprobe: &tetragon.ProcessKprobe{}}})
+	handleProcessedEvent(nil, &tetragon.GetEventsResponse{Event: &tetragon.GetEventsResponse_ProcessExec{ProcessExec: &tetragon.ProcessExec{}}})
+	handleProcessedEvent(nil, &tetragon.GetEventsResponse{Event: &tetragon.GetEventsResponse_ProcessTracepoint{ProcessTracepoint: &tetragon.ProcessTracepoint{}}})
+	handleProcessedEvent(nil, &tetragon.GetEventsResponse{Event: &tetragon.GetEventsResponse_ProcessExit{ProcessExit: &tetragon.ProcessExit{}}})
 
 	// empty pod
-	handleProcessedEvent(&tetragon.GetEventsResponse{Event: &tetragon.GetEventsResponse_ProcessKprobe{ProcessKprobe: &tetragon.ProcessKprobe{
+	handleProcessedEvent(nil, &tetragon.GetEventsResponse{Event: &tetragon.GetEventsResponse_ProcessKprobe{ProcessKprobe: &tetragon.ProcessKprobe{
 		Process: &tetragon.Process{Binary: "binary_a"},
 	}}})
-	handleProcessedEvent(&tetragon.GetEventsResponse{Event: &tetragon.GetEventsResponse_ProcessExec{ProcessExec: &tetragon.ProcessExec{
+	handleProcessedEvent(nil, &tetragon.GetEventsResponse{Event: &tetragon.GetEventsResponse_ProcessExec{ProcessExec: &tetragon.ProcessExec{
 		Process: &tetragon.Process{Binary: "binary_b"},
 	}}})
-	handleProcessedEvent(&tetragon.GetEventsResponse{Event: &tetragon.GetEventsResponse_ProcessTracepoint{ProcessTracepoint: &tetragon.ProcessTracepoint{
+	handleProcessedEvent(nil, &tetragon.GetEventsResponse{Event: &tetragon.GetEventsResponse_ProcessTracepoint{ProcessTracepoint: &tetragon.ProcessTracepoint{
 		Process: &tetragon.Process{Binary: "binary_c"},
 	}}})
-	handleProcessedEvent(&tetragon.GetEventsResponse{Event: &tetragon.GetEventsResponse_ProcessExit{ProcessExit: &tetragon.ProcessExit{
+	handleProcessedEvent(nil, &tetragon.GetEventsResponse{Event: &tetragon.GetEventsResponse_ProcessExit{ProcessExit: &tetragon.ProcessExit{
 		Process: &tetragon.Process{Binary: "binary_e"},
 	}}})
 
 	// with pod
-	handleProcessedEvent(&tetragon.GetEventsResponse{Event: &tetragon.GetEventsResponse_ProcessKprobe{ProcessKprobe: &tetragon.ProcessKprobe{
+	handleProcessedEvent(nil, &tetragon.GetEventsResponse{Event: &tetragon.GetEventsResponse_ProcessKprobe{ProcessKprobe: &tetragon.ProcessKprobe{
 		Process: &tetragon.Process{
 			Binary: "binary_a",
 			Pod:    &tetragon.Pod{Namespace: "namespace_a", Name: "pod_a"},
 		},
 	}}})
-	handleProcessedEvent(&tetragon.GetEventsResponse{Event: &tetragon.GetEventsResponse_ProcessExec{ProcessExec: &tetragon.ProcessExec{
+	handleProcessedEvent(nil, &tetragon.GetEventsResponse{Event: &tetragon.GetEventsResponse_ProcessExec{ProcessExec: &tetragon.ProcessExec{
 		Process: &tetragon.Process{
 			Binary: "binary_b",
 			Pod:    &tetragon.Pod{Namespace: "namespace_b", Name: "pod_b"},
 		},
 	}}})
-	handleProcessedEvent(&tetragon.GetEventsResponse{Event: &tetragon.GetEventsResponse_ProcessTracepoint{ProcessTracepoint: &tetragon.ProcessTracepoint{
+	handleProcessedEvent(nil, &tetragon.GetEventsResponse{Event: &tetragon.GetEventsResponse_ProcessTracepoint{ProcessTracepoint: &tetragon.ProcessTracepoint{
 		Process: &tetragon.Process{
 			Binary: "binary_c",
 			Pod:    &tetragon.Pod{Namespace: "namespace_c", Name: "pod_c"},
 		},
 	}}})
-	handleProcessedEvent(&tetragon.GetEventsResponse{Event: &tetragon.GetEventsResponse_ProcessExit{ProcessExit: &tetragon.ProcessExit{
+	handleProcessedEvent(nil, &tetragon.GetEventsResponse{Event: &tetragon.GetEventsResponse_ProcessExit{ProcessExit: &tetragon.ProcessExit{
 		Process: &tetragon.Process{
 			Binary: "binary_e",
 			Pod:    &tetragon.Pod{Namespace: "namespace_e", Name: "pod_e"},

pkg/sensors/tracing/generickprobe.go

@@ -114,6 +114,9 @@ type genericKprobe struct {
 	actionArgs idtable.Table
 
 	pinPathPrefix string
+
+	// policyName is the name of the policy that this tracepoint belongs to
+	policyName string
 }
 
 // pendingEvent is an event waiting to be merged with another event.
@@ -326,7 +329,12 @@ func flagsString(flags uint32) string {
 	return s
 }
 
-func createGenericKprobeSensor(name string, kprobes []v1alpha1.KProbeSpec, policyID policyfilter.PolicyID) (*sensors.Sensor, error) {
+func createGenericKprobeSensor(
+	name string,
+	kprobes []v1alpha1.KProbeSpec,
+	policyID policyfilter.PolicyID,
+	policyName string,
+) (*sensors.Sensor, error) {
 	var progs []*program.Program
 	var maps []*program.Map
 	var multiIDs, multiRetIDs []idtable.EntryID
@@ -479,6 +487,7 @@ func createGenericKprobeSensor(name string, kprobes []v1alpha1.KProbeSpec, polic
 			funcName:          funcName,
 			pendingEvents:     nil,
 			tableId:           idtable.UninitializedEntryID,
+			policyName:        policyName,
 		}
 
 		// Parse Filters into kernel filter logic
@@ -855,6 +864,7 @@ func handleGenericKprobe(r *bytes.Reader) ([]observer.Event, error) {
 	unix.FuncName = gk.funcName
 	unix.Namespaces = m.Namespaces
 	unix.Capabilities = m.Capabilities
+	unix.PolicyName = gk.policyName
 
 	returnEvent := m.Common.Flags > 0
 

pkg/sensors/tracing/generictracepoint.go

@@ -78,6 +78,9 @@ type genericTracepoint struct {
 	actionArgs idtable.Table
 
 	pinPathPrefix string
+
+	// policyName is the name of the policy that this tracepoint belongs to
+	policyName string
 }
 
 // genericTracepointArg is the internal representation of an output value of a
@@ -293,7 +296,12 @@ func buildGenericTracepointArgs(info *tracepoint.Tracepoint, specArgs []v1alpha1
 
 // createGenericTracepoint creates the genericTracepoint information based on
 // the user-provided configuration
-func createGenericTracepoint(sensorName string, conf *GenericTracepointConf, policyID policyfilter.PolicyID) (*genericTracepoint, error) {
+func createGenericTracepoint(
+	sensorName string,
+	conf *GenericTracepointConf,
+	policyID policyfilter.PolicyID,
+	policyName string,
+) (*genericTracepoint, error) {
 	tp := tracepoint.Tracepoint{
 		Subsys: conf.Subsystem,
 		Event:  conf.Event,
@@ -309,10 +317,11 @@ func createGenericTracepoint(sensorName string, conf *GenericTracepointConf, pol
 	}
 
 	ret := &genericTracepoint{
-		Info:     &tp,
-		Spec:     conf,
-		args:     tpArgs,
-		policyID: policyID,
+		Info:       &tp,
+		Spec:       conf,
+		args:       tpArgs,
+		policyID:   policyID,
+		policyName: policyName,
 	}
 
 	genericTracepointTable.addTracepoint(ret)
@@ -321,11 +330,16 @@ func createGenericTracepoint(sensorName string, conf *GenericTracepointConf, pol
 }
 
 // createGenericTracepointSensor will create a sensor that can be loaded based on a generic tracepoint configuration
-func createGenericTracepointSensor(name string, confs []GenericTracepointConf, policyID policyfilter.PolicyID) (*sensors.Sensor, error) {
+func createGenericTracepointSensor(
+	name string,
+	confs []GenericTracepointConf,
+	policyID policyfilter.PolicyID,
+	policyName string,
+) (*sensors.Sensor, error) {
 
 	tracepoints := make([]*genericTracepoint, 0, len(confs))
 	for i := range confs {
-		tp, err := createGenericTracepoint(name, &confs[i], policyID)
+		tp, err := createGenericTracepoint(name, &confs[i], policyID, policyName)
 		if err != nil {
 			return nil, err
 		}
@@ -604,6 +618,7 @@ func handleGenericTracepoint(r *bytes.Reader) ([]observer.Event, error) {
 
 	unix.Subsys = tp.Info.Subsys
 	unix.Event = tp.Info.Event
+	unix.PolicyName = tp.policyName
 
 	for idx, out := range tp.args {
 

pkg/sensors/tracing/genericuprobe.go

@@ -44,6 +44,8 @@ type genericUprobe struct {
 	path          string
 	symbol        string
 	selectors     *selectors.KernelSelectorState
+	// policyName is the name of the policy that this uprobe belongs to
+	policyName string
 }
 
 func (g *genericUprobe) SetID(id idtable.EntryID) {
@@ -169,7 +171,11 @@ func isValidUprobeSelectors(selectors []v1alpha1.KProbeSelector) error {
 	return nil
 }
 
-func createGenericUprobeSensor(name string, uprobes []v1alpha1.UProbeSpec) (*sensors.Sensor, error) {
+func createGenericUprobeSensor(
+	name string,
+	uprobes []v1alpha1.UProbeSpec,
+	policyName string,
+) (*sensors.Sensor, error) {
 	var progs []*program.Program
 	var maps []*program.Map
 
@@ -199,11 +205,12 @@ func createGenericUprobeSensor(name string, uprobes []v1alpha1.UProbeSpec) (*sen
 		}
 
 		uprobeEntry := &genericUprobe{
-			tableId:   idtable.UninitializedEntryID,
-			config:    config,
-			path:      spec.Path,
-			symbol:    spec.Symbol,
-			selectors: uprobeSelectorState,
+			tableId:    idtable.UninitializedEntryID,
+			config:     config,
+			path:       spec.Path,
+			symbol:     spec.Symbol,
+			selectors:  uprobeSelectorState,
+			policyName: policyName,
 		}
 
 		uprobeTable.AddEntry(uprobeEntry)
@@ -264,5 +271,6 @@ func (k *observerUprobeSensor) PolicyHandler(
 	}
 
 	name := fmt.Sprintf("gup-sensor-%d", atomic.AddUint64(&sensorCounter, 1))
-	return createGenericUprobeSensor(name, spec.UProbes)
+	policyName := p.TpName()
+	return createGenericUprobeSensor(name, spec.UProbes, policyName)
 }

pkg/sensors/tracing/policyhandler.go

@@ -21,6 +21,7 @@ func (h policyHandler) PolicyHandler(
 	policyID policyfilter.PolicyID,
 ) (*sensors.Sensor, error) {
 
+	policyName := policy.TpName()
 	spec := policy.TpSpec()
 	if len(spec.KProbes) > 0 && len(spec.Tracepoints) > 0 {
 		return nil, errors.New("tracing policies with both kprobes and tracepoints are not currently supported")
@@ -31,11 +32,11 @@ func (h policyHandler) PolicyHandler(
 		if err != nil {
 			return nil, err
 		}
-		return createGenericKprobeSensor(name, spec.KProbes, policyID)
+		return createGenericKprobeSensor(name, spec.KProbes, policyID, policyName)
 	}
 	if len(spec.Tracepoints) > 0 {
 		name := fmt.Sprintf("gtp-sensor-%d", atomic.AddUint64(&sensorCounter, 1))
-		return createGenericTracepointSensor(name, spec.Tracepoints, policyID)
+		return createGenericTracepointSensor(name, spec.Tracepoints, policyID, policyName)
 	}
 	return nil, nil
 }

pkg/sensors/tracing/tracepoint_test.go

@@ -20,6 +20,7 @@ import (
 	lc "github.com/cilium/tetragon/pkg/matchers/listmatcher"
 	smatcher "github.com/cilium/tetragon/pkg/matchers/stringmatcher"
 	"github.com/cilium/tetragon/pkg/observer"
+	"github.com/cilium/tetragon/pkg/policyfilter"
 	"github.com/cilium/tetragon/pkg/sensors"
 	testsensor "github.com/cilium/tetragon/pkg/sensors/test"
 	tus "github.com/cilium/tetragon/pkg/testutils/sensors"
@@ -65,7 +66,7 @@ func TestGenericTracepointSimple(t *testing.T) {
 
 	sm := tus.StartTestSensorManager(ctx, t)
 	// create and add sensor
-	sensor, err := createGenericTracepointSensor("GtpLseekTest", []GenericTracepointConf{lseekConf}, 0)
+	sensor, err := createGenericTracepointSensor("GtpLseekTest", []GenericTracepointConf{lseekConf}, policyfilter.NoFilterID, "policyName")
 	if err != nil {
 		t.Fatalf("failed to create generic tracepoint sensor: %s", err)
 	}
@@ -124,7 +125,7 @@ func doTestGenericTracepointPidFilter(t *testing.T, conf GenericTracepointConf,
 
 	sm := tus.StartTestSensorManager(ctx, t)
 	// create and add sensor
-	sensor, err := createGenericTracepointSensor("GtpLseekTest", []GenericTracepointConf{conf}, 0)
+	sensor, err := createGenericTracepointSensor("GtpLseekTest", []GenericTracepointConf{conf}, policyfilter.NoFilterID, "policyName")
 	if err != nil {
 		t.Fatalf("failed to create generic tracepoint sensor: %s", err)
 	}