Replace AP and NN cache with CP

Makefile

@@ -11,6 +11,10 @@ TAG?=test
 binary:
 	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o $(BINARY_NAME) ./cmd/main.go
 
+.PHONY: check-legacy-packages
+check-legacy-packages:
+	go test ./tests/containerprofilecache -run TestLegacyPackagesDeleted
+
 docker-build-only:
 	docker buildx build --platform linux/amd64 -t $(IMAGE):$(TAG) -f $(DOCKERFILE_PATH) --load .
 

cmd/main.go

@@ -45,10 +45,9 @@ import (
 	"github.com/kubescape/node-agent/pkg/nodeprofilemanager"
 	nodeprofilemanagerv1 "github.com/kubescape/node-agent/pkg/nodeprofilemanager/v1"
 	"github.com/kubescape/node-agent/pkg/objectcache"
-	"github.com/kubescape/node-agent/pkg/objectcache/applicationprofilecache"
+	"github.com/kubescape/node-agent/pkg/objectcache/containerprofilecache"
 	"github.com/kubescape/node-agent/pkg/objectcache/dnscache"
 	"github.com/kubescape/node-agent/pkg/objectcache/k8scache"
-	"github.com/kubescape/node-agent/pkg/objectcache/networkneighborhoodcache"
 	objectcachev1 "github.com/kubescape/node-agent/pkg/objectcache/v1"
 	"github.com/kubescape/node-agent/pkg/processtree"
 	containerprocesstree "github.com/kubescape/node-agent/pkg/processtree/container"
@@ -297,16 +296,14 @@ func main() {
 		ruleBindingNotify = make(chan rulebinding.RuleBindingNotify, 100)
 		ruleBindingCache.AddNotifier(&ruleBindingNotify)
 
-		apc := applicationprofilecache.NewApplicationProfileCache(cfg, storageClient, k8sObjectCache)
-		apc.Start(ctx)
-
-		nnc := networkneighborhoodcache.NewNetworkNeighborhoodCache(cfg, storageClient, k8sObjectCache)
-		nnc.Start(ctx)
+		cpc := containerprofilecache.NewContainerProfileCache(cfg, storageClient, k8sObjectCache, prometheusExporter)
+		cpc.Start(ctx)
+		logger.L().Info("ContainerProfileCache active; legacy AP/NN caches removed")
 
 		dc := dnscache.NewDnsCache(dnsResolver)
 
 		// create object cache
-		objCache = objectcachev1.NewObjectCache(k8sObjectCache, apc, nnc, dc)
+		objCache = objectcachev1.NewObjectCache(k8sObjectCache, cpc, dc)
 
 		ruleCooldown := rulecooldown.NewRuleCooldown(cfg.RuleCoolDown)
 
@@ -328,10 +325,9 @@ func main() {
 
 	} else {
 		ruleManager = rulemanager.CreateRuleManagerMock()
-		apc := &objectcache.ApplicationProfileCacheMock{}
-		nnc := &objectcache.NetworkNeighborhoodCacheMock{}
+		cpc := &objectcache.ContainerProfileCacheMock{}
 		dc := &objectcache.DnsCacheMock{}
-		objCache = objectcachev1.NewObjectCache(k8sObjectCache, apc, nnc, dc)
+		objCache = objectcachev1.NewObjectCache(k8sObjectCache, cpc, dc)
 		ruleBindingNotify = make(chan rulebinding.RuleBindingNotify, 1)
 	}
 

pkg/config/config.go

@@ -106,6 +106,7 @@ type Config struct {
 	ProcfsPidScanInterval          time.Duration                        `mapstructure:"procfsPidScanInterval"`
 	ProcfsScanInterval             time.Duration                        `mapstructure:"procfsScanInterval"`
 	ProfilesCacheRefreshRate       time.Duration                        `mapstructure:"profilesCacheRefreshRate"`
+	StorageRPCBudget               time.Duration                        `mapstructure:"storageRPCBudget"`
 	RuleCoolDown                   rulecooldown.RuleCooldownConfig      `mapstructure:"ruleCooldown"`
 	TestMode                       bool                                 `mapstructure:"testMode"`
 	UpdateDataPeriod               time.Duration                        `mapstructure:"updateDataPeriod"`

pkg/containerwatcher/v2/container_watcher_collection.go

@@ -60,8 +60,7 @@ func (cw *ContainerWatcher) StartContainerCollection(ctx context.Context) error
 		cw.containerCallbackAsync,
 		cw.containerProcessTree.ContainerCallback,
 		cw.containerProfileManager.ContainerCallback,
-		cw.objectCache.ApplicationProfileCache().ContainerCallback,
-		cw.objectCache.NetworkNeighborhoodCache().ContainerCallback,
+		cw.objectCache.ContainerProfileCache().ContainerCallback,
 		cw.malwareManager.ContainerCallback,
 		cw.ruleManager.ContainerCallback,
 		cw.sbomManager.ContainerCallback,

pkg/metricsmanager/metrics_manager_interface.go

@@ -20,4 +20,9 @@ type MetricsManager interface {
 	ReportContainerStart()
 	ReportContainerStop()
 	ReportDedupEvent(eventType utils.EventType, duplicate bool)
+	ReportContainerProfileLegacyLoad(kind, completeness string)
+	SetContainerProfileCacheEntries(kind string, count float64)
+	ReportContainerProfileCacheHit(hit bool)
+	ReportContainerProfileReconcilerDuration(phase string, duration time.Duration)
+	ReportContainerProfileReconcilerEviction(reason string)
 }

pkg/metricsmanager/metrics_manager_mock.go

@@ -66,4 +66,9 @@ func (m *MetricsMock) ReportContainerStart() {}
 
 func (m *MetricsMock) ReportContainerStop() {}
 
-func (m *MetricsMock) ReportDedupEvent(eventType utils.EventType, duplicate bool) {}
+func (m *MetricsMock) ReportDedupEvent(eventType utils.EventType, duplicate bool)          {}
+func (m *MetricsMock) ReportContainerProfileLegacyLoad(_, _ string)                       {}
+func (m *MetricsMock) SetContainerProfileCacheEntries(_ string, _ float64)                {}
+func (m *MetricsMock) ReportContainerProfileCacheHit(_ bool)                              {}
+func (m *MetricsMock) ReportContainerProfileReconcilerDuration(_ string, _ time.Duration) {}
+func (m *MetricsMock) ReportContainerProfileReconcilerEviction(_ string)                  {}

pkg/metricsmanager/metrics_manager_noop.go

@@ -22,3 +22,8 @@ func (m *MetricsNoop) ReportRuleEvaluationTime(_ string, _ utils.EventType, _ ti
 func (m *MetricsNoop) ReportContainerStart()                                                {}
 func (m *MetricsNoop) ReportContainerStop()                                                 {}
 func (m *MetricsNoop) ReportDedupEvent(_ utils.EventType, _ bool)                           {}
+func (m *MetricsNoop) ReportContainerProfileLegacyLoad(_, _ string)                        {}
+func (m *MetricsNoop) SetContainerProfileCacheEntries(_ string, _ float64)                 {}
+func (m *MetricsNoop) ReportContainerProfileCacheHit(_ bool)                               {}
+func (m *MetricsNoop) ReportContainerProfileReconcilerDuration(_ string, _ time.Duration)  {}
+func (m *MetricsNoop) ReportContainerProfileReconcilerEviction(_ string)                   {}

pkg/metricsmanager/prometheus/prometheus.go

@@ -63,6 +63,13 @@ type PrometheusMetric struct {
 	// Dedup metrics
 	dedupEventCounter *prometheus.CounterVec
 
+	// ContainerProfile cache metrics
+	cpCacheLegacyLoadsCounter      *prometheus.CounterVec
+	cpCacheEntriesGauge            *prometheus.GaugeVec
+	cpCacheHitCounter              *prometheus.CounterVec
+	cpReconcilerDurationHistogram  *prometheus.HistogramVec
+	cpReconcilerEvictionsCounter   *prometheus.CounterVec
+
 	// Cache to avoid allocating Labels maps on every call
 	ruleCounterCache          map[string]prometheus.Counter
 	rulePrefilteredCounterCache map[string]prometheus.Counter
@@ -215,6 +222,29 @@ func NewPrometheusMetric() *PrometheusMetric {
 			Help: "Total number of events processed by the dedup layer",
 		}, []string{eventTypeLabel, "result"}),
 
+		// ContainerProfile cache metrics
+		cpCacheLegacyLoadsCounter: promauto.NewCounterVec(prometheus.CounterOpts{
+			Name: "node_agent_user_profile_legacy_loads_total",
+			Help: "Number of times a user-authored legacy ApplicationProfile or NetworkNeighborhood was loaded into the ContainerProfileCache; will be removed in a future release.",
+		}, []string{"kind", "completeness"}),
+		cpCacheEntriesGauge: promauto.NewGaugeVec(prometheus.GaugeOpts{
+			Name: "node_agent_containerprofile_cache_entries",
+			Help: "Current number of cached ContainerProfile entries per kind.",
+		}, []string{"kind"}),
+		cpCacheHitCounter: promauto.NewCounterVec(prometheus.CounterOpts{
+			Name: "node_agent_containerprofile_cache_hit_total",
+			Help: "Total number of ContainerProfile cache lookups by result.",
+		}, []string{"result"}),
+		cpReconcilerDurationHistogram: promauto.NewHistogramVec(prometheus.HistogramOpts{
+			Name:    "node_agent_containerprofile_reconciler_duration_seconds",
+			Help:    "Duration of ContainerProfile reconciler phases in seconds.",
+			Buckets: prometheus.DefBuckets,
+		}, []string{"phase"}),
+		cpReconcilerEvictionsCounter: promauto.NewCounterVec(prometheus.CounterOpts{
+			Name: "node_agent_containerprofile_reconciler_evictions_total",
+			Help: "Total number of ContainerProfile cache evictions by reason.",
+		}, []string{"reason"}),
+
 		// Initialize counter caches
 		ruleCounterCache:            make(map[string]prometheus.Counter),
 		rulePrefilteredCounterCache: make(map[string]prometheus.Counter),
@@ -256,6 +286,11 @@ func (p *PrometheusMetric) Destroy() {
 	prometheus.Unregister(p.containerStartCounter)
 	prometheus.Unregister(p.containerStopCounter)
 	prometheus.Unregister(p.dedupEventCounter)
+	prometheus.Unregister(p.cpCacheLegacyLoadsCounter)
+	prometheus.Unregister(p.cpCacheEntriesGauge)
+	prometheus.Unregister(p.cpCacheHitCounter)
+	prometheus.Unregister(p.cpReconcilerDurationHistogram)
+	prometheus.Unregister(p.cpReconcilerEvictionsCounter)
 	// Unregister program ID metrics
 	prometheus.Unregister(p.programRuntimeGauge)
 	prometheus.Unregister(p.programRunCountGauge)
@@ -432,3 +467,27 @@ func (p *PrometheusMetric) ReportDedupEvent(eventType utils.EventType, duplicate
 	}
 	p.dedupEventCounter.WithLabelValues(string(eventType), result).Inc()
 }
+
+func (p *PrometheusMetric) ReportContainerProfileLegacyLoad(kind, completeness string) {
+	p.cpCacheLegacyLoadsCounter.WithLabelValues(kind, completeness).Inc()
+}
+
+func (p *PrometheusMetric) SetContainerProfileCacheEntries(kind string, count float64) {
+	p.cpCacheEntriesGauge.WithLabelValues(kind).Set(count)
+}
+
+func (p *PrometheusMetric) ReportContainerProfileCacheHit(hit bool) {
+	result := "hit"
+	if !hit {
+		result = "miss"
+	}
+	p.cpCacheHitCounter.WithLabelValues(result).Inc()
+}
+
+func (p *PrometheusMetric) ReportContainerProfileReconcilerDuration(phase string, duration time.Duration) {
+	p.cpReconcilerDurationHistogram.WithLabelValues(phase).Observe(duration.Seconds())
+}
+
+func (p *PrometheusMetric) ReportContainerProfileReconcilerEviction(reason string) {
+	p.cpReconcilerEvictionsCounter.WithLabelValues(reason).Inc()
+}