feat: support for configuring memory limit in cluster collector (#1956)

Co-authored-by: Ben Elferink <[email protected]> Co-authored-by: Tamir David <[email protected]>
odigos-io · Dec 9, 2024 · 6ec613c · 6ec613c
1 parent 43e210a
commit 6ec613c
Show file tree

Hide file tree

Showing 6 changed files with 24 additions and 2 deletions.
diff --git a/cli/cmd/resources/odigosconfig.go b/cli/cmd/resources/odigosconfig.go
@@ -74,6 +74,7 @@ func GetGatewayConfigBasedOnSize(profile common.ProfileName) *common.CollectorGa
 				RequestCPUm:      150,
 				LimitCPUm:        300,
 				RequestMemoryMiB: 300,
+				LimitMemoryMiB:   300,
 			}
 		case sizeMProfile.ProfileName:
 			return &common.CollectorGatewayConfiguration{
@@ -82,6 +83,7 @@ func GetGatewayConfigBasedOnSize(profile common.ProfileName) *common.CollectorGa
 				RequestCPUm:      500,
 				LimitCPUm:        1000,
 				RequestMemoryMiB: 500,
+				LimitMemoryMiB:   600,
 			}
 		case sizeLProfile.ProfileName:
 			return &common.CollectorGatewayConfiguration{
@@ -90,6 +92,7 @@ func GetGatewayConfigBasedOnSize(profile common.ProfileName) *common.CollectorGa
 				RequestCPUm:      750,
 				LimitCPUm:        1250,
 				RequestMemoryMiB: 750,
+				LimitMemoryMiB:   850,
 			}
 		}
 	}

diff --git a/common/odigos_config.go b/common/odigos_config.go
@@ -21,6 +21,11 @@ type CollectorGatewayConfiguration struct {
 	// default value is 500Mi
 	RequestMemoryMiB int `json:"requestMemoryMiB,omitempty"`
 
+	// LimitMemoryMiB is the memory limit for the cluster gateway collector deployment.
+	// it will be embedded in the deployment as a resource limit of the form "memory: <value>Mi"
+	// default value is 1.25 the memory request.
+	LimitMemoryMiB int `json:"limitMemoryMiB,omitempty"`
+
 	// RequestCPUm is the CPU request for the cluster gateway collector deployment.
 	// it will be embedded in the deployment as a resource request of the form "cpu: <value>m"
 	// default value is 500m

diff --git a/docs/pipeline/configuration.mdx b/docs/pipeline/configuration.mdx
@@ -53,6 +53,10 @@ collectorGateway:
   # This value will be embedded in the deployment as a resource request of the form "memory: <value>Mi".
   requestMemoryMiB: 500
 
+  # LimitMemoryMiB is the memory limit for the cluster gateway collector deployment.
+  # This value will be embedded in the deployment as a resource limit of the form "memory: <value>Mi".
+  limitMemoryMiB: 625
+
   # RequestCPUm is the CPU request for the cluster gateway collector deployment.
   # This value will be embedded in the deployment as a resource request of the form "cpu: <value>m".
   requestCPUm: 500

diff --git a/helm/odigos/templates/odigos-config-cm.yaml b/helm/odigos/templates/odigos-config-cm.yaml
@@ -26,6 +26,9 @@ data:
       {{- with .Values.collectorGateway.requestMemoryMiB }}
       requestMemoryMiB: {{ . }}
       {{- end }}
+      {{- with .Values.collectorGateway.limitMemoryMiB }}
+      limitMemoryMiB: {{ . }}
+      {{- end }}
       {{- with .Values.collectorGateway.requestCPUm }}
       requestCPUm: {{ . }}
       {{- end }}

diff --git a/helm/odigos/values.yaml b/helm/odigos/values.yaml
@@ -25,6 +25,11 @@ collectorGateway:
   # of the form "memory: <value>Mi".
   # default value is 500Mi
   requestMemoryMiB: 500
+  # the memory limit for the cluster gateway collector deployment.
+  # it will be embedded in the deployment as a resource limit
+  # of the form "memory: <value>Mi".
+  # default value is 625Mi
+  limitMemoryMiB: 625
 
   # the CPU request for the cluster gateway collector deployment.
   # it will be embedded in the deployment as a resource request

diff --git a/scheduler/controllers/clustercollectorsgroup/resource_config.go b/scheduler/controllers/clustercollectorsgroup/resource_config.go
@@ -55,6 +55,10 @@ func getGatewayResourceSettings(odigosConfig *common.OdigosConfiguration) *odigo
 	if gatewayConfig != nil && gatewayConfig.RequestMemoryMiB > 0 {
 		memoryRequestMiB = gatewayConfig.RequestMemoryMiB
 	}
+	memoryLimitMiB := int(float64(memoryRequestMiB) * memoryLimitAboveRequestFactor)
+	if odigosConfig.CollectorGateway != nil && odigosConfig.CollectorGateway.LimitMemoryMiB > 0 {
+		memoryLimitMiB = odigosConfig.CollectorGateway.LimitMemoryMiB
+	}
 	cpuRequestm := defaultRequestCPUm
 	if gatewayConfig != nil && gatewayConfig.RequestCPUm > 0 {
 		cpuRequestm = gatewayConfig.RequestCPUm
@@ -75,8 +79,6 @@ func getGatewayResourceSettings(odigosConfig *common.OdigosConfiguration) *odigo
 		memoryLimiterSpikeLimitMiB = odigosConfig.CollectorGateway.MemoryLimiterSpikeLimitMiB
 	}
 
-	memoryLimitMiB := int(float64(memoryRequestMiB) * memoryLimitAboveRequestFactor)
-
 	gomemlimitMiB := int(memoryLimiterLimitMiB * defaultGoMemLimitPercentage / 100.0)
 	if odigosConfig.CollectorGateway != nil && odigosConfig.CollectorGateway.GoMemLimitMib != 0 {
 		gomemlimitMiB = odigosConfig.CollectorGateway.GoMemLimitMib