Wiping option for MongoDB resources during clusters ingestion (KHaaS only)

configs/etc/kubehound-reference.yaml

@@ -54,6 +54,9 @@ mongodb:
   # Timeout on requests to the mongo DB instance
   connection_timeout: 30s
 
+  # Wipe old copy of k8s/kubehound resources in mongodb
+  wipe: false
+
 # Graph database configuration
 janusgraph:
   # Connection URL to the JanusGraph DB instance

configs/etc/kubehound.yaml

@@ -28,7 +28,10 @@ mongodb:
 
   # Timeout on requests to the mongo DB instance
   connection_timeout: 30s
-
+
+  # Wipe old copy of k8s/kubehound resources in mongodb
+  wipe: false
+
 # Graph database configuration
 janusgraph:
   # Connection URL to the JanusGraph DB instance

deployments/k8s/khaas/conf/ingestor/kubehound.yaml

@@ -43,6 +43,9 @@ janusgraph:
   # Timeout on requests to the JanusGraph DB instance
   connection_timeout: 30s
 
+  # Wipe old copy of k8s/kubehound resources in mongodb
+  wipe: false
+
 # Datadog telemetry configuration
 telemetry:
   # Whether to enable Datadog telemetry (default false)

pkg/config/config.go

@@ -116,6 +116,7 @@ func SetDefaultValues(ctx context.Context, v *viper.Viper) {
 	// Default value for MongoDB
 	v.SetDefault(MongoUrl, DefaultMongoUrl)
 	v.SetDefault(MongoConnectionTimeout, DefaultConnectionTimeout)
+	v.SetDefault(MongoWipe, DefaultMongoWipe)
 
 	// Defaults values for JanusGraph
 	v.SetDefault(JanusGraphUrl, DefaultJanusGraphUrl)

pkg/config/mongodb.go

@@ -5,14 +5,17 @@ import (
 )
 
 const (
-	DefaultMongoUrl = "mongodb://localhost:27017"
+	DefaultMongoUrl  = "mongodb://localhost:27017"
+	DefaultMongoWipe = false
 
 	MongoUrl               = "mongodb.url"
 	MongoConnectionTimeout = "mongodb.connection_timeout"
+	MongoWipe              = "mongodb.wipe"
 )
 
 // MongoDBConfig configures mongodb specific parameters.
 type MongoDBConfig struct {
 	URL               string        `mapstructure:"url"` // Mongodb specific configuration
 	ConnectionTimeout time.Duration `mapstructure:"connection_timeout"`
+	Wipe              bool          `mapstructure:"wipe"`
 }

pkg/ingestor/api/api.go

@@ -211,11 +211,23 @@ func (g *IngestorAPI) Ingest(ctx context.Context, path string) error { //nolint:
 
 	// Run the ingest pipeline
 	l.Info("Starting Kubernetes raw data ingest")
+
+	// Droping the storedb data for the cluster if the wipe flag is set
+	if g.Cfg.MongoDB.Wipe {
+		err = g.providers.StoreProvider.Clean(runCtx, "*", clusterName) //nolint: contextcheck
+		if err != nil {
+			return err
+		}
+		l.Info("Droped storedb data for the cluster", log.String(log.FieldClusterKey, clusterName))
+	}
+
+	// Checking if the data is already ingested in the database
 	alreadyIngestedInDB, err := g.isAlreadyIngestedInDB(runCtx, clusterName, runID) //nolint: contextcheck
 	if err != nil {
 		return err
 	}
 
+	// Droping the storedb data for the cluster if the data is already ingested in the database
 	if alreadyIngestedInDB {
 		l.Info("Data already ingested in the database for %s/%s, droping the current data", log.String(log.FieldClusterKey, clusterName), log.String(log.FieldRunIDKey, runID))
 		err := g.providers.StoreProvider.Clean(runCtx, runID, clusterName) //nolint: contextcheck