feat: include memory limit in memory settings (#1825)

For cluster collector, we currently populate the resource request for
memory with default value of `500MiB`.

However, we do not set a limit on the resource request.

This PR adds a limit for memory request for the cluster collector, set
by default to `1.25` times the memory request, `625MiB` by default. It
is not expected to be reached, but set as protection to guarantee we
supply a small slack for extreme bursts, but capping the memory at a
reasonable size to protect the cluster

api/config/crd/bases/odigos.io_collectorsgroups.yaml

@@ -62,6 +62,14 @@ spec:
                       this is when go runtime will start garbage collection.
                       it is recommended to be set to 80% of the hard limit of the memory limiter.
                     type: integer
+                  memoryLimitMiB:
+                    description: |-
+                      This option sets the limit on the memory usage of the collector.
+                      since the memory limiter mechanism is heuristic, and operates on fixed intervals,
+                      while it cannot fully prevent OOMs, it can help in reducing the chances of OOMs in edge cases.
+                      the settings should prevent the collector from exceeding the memory request,
+                      so one can set this to the same value as the memory request or higher to allow for some buffer for bursts.
+                    type: integer
                   memoryLimiterLimitMiB:
                     description: |-
                       this parameter sets the "limit_mib" parameter in the memory limiter configuration for the collector.
@@ -86,6 +94,7 @@ spec:
                     type: integer
                 required:
                 - gomemlimitMiB
+                - memoryLimitMiB
                 - memoryLimiterLimitMiB
                 - memoryLimiterSpikeLimitMiB
                 - memoryRequestMiB

api/odigos/v1alpha1/collectorsgroup_types.go

@@ -40,6 +40,13 @@ type CollectorsGroupMemorySettings struct {
 	// it will be embedded in the as a resource request of the form "memory: <value>Mi"
 	MemoryRequestMiB int `json:"memoryRequestMiB"`
 
+	// This option sets the limit on the memory usage of the collector.
+	// since the memory limiter mechanism is heuristic, and operates on fixed intervals,
+	// while it cannot fully prevent OOMs, it can help in reducing the chances of OOMs in edge cases.
+	// the settings should prevent the collector from exceeding the memory request,
+	// so one can set this to the same value as the memory request or higher to allow for some buffer for bursts.
+	MemoryLimitMiB int `json:"memoryLimitMiB"`
+
 	// this parameter sets the "limit_mib" parameter in the memory limiter configuration for the collector.
 	// it is the hard limit after which a force garbage collection will be performed.
 	// this value will end up comparing against the go runtime reported heap Alloc value.

autoscaler/controllers/gateway/deployment.go

@@ -91,6 +91,7 @@ func getDesiredDeployment(dests *odigosv1.DestinationList, configDataHash string
 	gateway *odigosv1.CollectorsGroup, scheme *runtime.Scheme, imagePullSecrets []string, odigosVersion string) (*appsv1.Deployment, error) {
 
 	requestMemoryQuantity := resource.MustParse(fmt.Sprintf("%dMi", gateway.Spec.MemorySettings.MemoryRequestMiB))
+	limitMemoryQuantity := resource.MustParse(fmt.Sprintf("%dMi", gateway.Spec.MemorySettings.MemoryLimitMiB))
 
 	desiredDeployment := &appsv1.Deployment{
 		ObjectMeta: v1.ObjectMeta{
@@ -190,6 +191,9 @@ func getDesiredDeployment(dests *odigosv1.DestinationList, configDataHash string
 								Requests: corev1.ResourceList{
 									corev1.ResourceMemory: requestMemoryQuantity,
 								},
+								Limits: corev1.ResourceList{
+									corev1.ResourceMemory: limitMemoryQuantity,
+								},
 							},
 						},
 					},

scheduler/controllers/clustercollectorsgroup/memory.go

@@ -19,6 +19,12 @@ const (
 	// the percentage out of the memory limiter hard limit, at which go runtime will start garbage collection.
 	// it is used to calculate the GOMEMLIMIT environment variable value.
 	defaultGoMemLimitPercentage = 80.0
+
+	// the memory settings should prevent the collector from exceeding the memory request.
+	// however, the mechanism is heuristic and does not guarantee to prevent OOMs.
+	// allowing the memory limit to be slightly above the memory request can help in reducing the chances of OOMs in edge cases.
+	// instead of having the process killed, it can use extra memory available on the node without allocating it preemptively.
+	memoryLimitAboveRequestFactor = 1.25
 )
 
 // process the memory settings from odigos config and return the memory settings for the collectors group.
@@ -29,6 +35,8 @@ func getMemorySettings(odigosConfig *common.OdigosConfiguration) *odigosv1.Colle
 		memoryRequestMiB = odigosConfig.CollectorGateway.RequestMemoryMiB
 	}
 
+	memoryLimitMiB := int(float64(memoryRequestMiB) * memoryLimitAboveRequestFactor)
+
 	// the memory limiter hard limit is set as 50 MiB less than the memory request
 	memoryLimiterLimitMiB := memoryRequestMiB - defaultMemoryLimiterLimitDiffMib
 	if odigosConfig.CollectorGateway != nil && odigosConfig.CollectorGateway.MemoryLimiterLimitMiB > 0 {
@@ -47,6 +55,7 @@ func getMemorySettings(odigosConfig *common.OdigosConfiguration) *odigosv1.Colle
 
 	return &odigosv1.CollectorsGroupMemorySettings{
 		MemoryRequestMiB:           memoryRequestMiB,
+		MemoryLimitMiB:             memoryLimitMiB,
 		MemoryLimiterLimitMiB:      memoryLimiterLimitMiB,
 		MemoryLimiterSpikeLimitMiB: memoryLimiterSpikeLimitMiB,
 		GomemlimitMiB:              gomemlimitMiB,