feat(helm): Make all container images configurable in values.yaml (resolves #2047).

docs/src/user-docs/guides-k8s-deployment.md

@@ -195,6 +195,27 @@ image:
     pullPolicy: "Never"  # Use "Never" for local images, "IfNotPresent" for remote
     tag: "latest"
 
+  # Override third-party container images (useful for private registries or AWS Marketplace ECR).
+  # See the chart's values.yaml for the full list of configurable images.
+  mariadb:
+    repository: "mariadb"
+    tag: "10.11.16"
+  mysql:
+    repository: "mysql"
+    tag: "8.0.46"
+  queue:
+    repository: "rabbitmq"
+    tag: "4.2.6"
+  redis:
+    repository: "redis"
+    tag: "7.4.8"
+  resultsCache:
+    repository: "mongo"
+    tag: "8.0.21"
+  kubectl:
+    repository: "bitnami/kubectl"
+    digest: "sha256:98736aabcecb8d3cbcdcd7b132d14b1d67ed99bac2f06d471f06235933103df3"  # v1.36.0
+
 # Adjust worker concurrency
 workerConcurrency: 16
 

tools/deployment/package-helm/.set-up-common.sh

@@ -45,11 +45,13 @@ prepare_environment() {
 # flags for using it. If image is not specified, returns empty string.
 #
 # @param {string} cluster_name Name of the kind cluster
+# @param {string} component Flat image component name (e.g., "clpPackage", "redis", "queue")
 # @param {string} [image] Docker image (e.g., "clp-package:dev-junhao-a6bf")
 # @return Prints helm --set flags to stdout
 get_image_helm_args() {
     local cluster_name=$1
-    local image="${2:-}"
+    local component=$2
+    local image="${3:-}"
 
     if [[ -z "${image}" ]]; then
         return
@@ -67,9 +69,9 @@ get_image_helm_args() {
         echo "Error: '${image}' is not a valid image reference (expected repo:tag)." >&2
         return 1
     fi
-    echo "--set" "image.clpPackage.repository=${repo}" \
-         "--set" "image.clpPackage.tag=${tag}" \
-         "--set" "image.clpPackage.pullPolicy=Never"
+    echo "--set" "image.${component}.repository=${repo}" \
+         "--set" "image.${component}.tag=${tag}" \
+         "--set" "image.${component}.pullPolicy=Never"
 }
 
 # Parses common arguments shared across set-up scripts.

tools/deployment/package-helm/Chart.yaml

@@ -1,6 +1,6 @@
 apiVersion: "v2"
 name: "clp"
-version: "0.3.2-dev.9"
+version: "0.3.2-dev.10"
 description: "A Helm chart for CLP's (Compressed Log Processor) package deployment"
 type: "application"
 appVersion: "0.12.1-dev"

tools/deployment/package-helm/set-up-multi-dedicated-test.sh

@@ -161,6 +161,6 @@ helm install test "${script_dir}" \
     --set "scheduling.webui.nodeSelector.yscope\.io/nodeType=core" \
     --set "scheduling.mcpServer.nodeSelector.yscope\.io/nodeType=core" \
     $(get_presto_helm_args) \
-    $(get_image_helm_args "${CLUSTER_NAME}" "${CLP_PACKAGE_IMAGE}")
+    $(get_image_helm_args "${CLUSTER_NAME}" "clpPackage" "${CLP_PACKAGE_IMAGE}")
 
 wait_for_cluster_ready

tools/deployment/package-helm/set-up-multi-shared-test.sh

@@ -51,6 +51,6 @@ helm install test "${script_dir}" \
     --set "scheduling.reducer.replicas=${REDUCER_REPLICAS}" \
     --set "scheduling.prestoWorker.replicas=${PRESTO_WORKER_REPLICAS}" \
     $(get_presto_helm_args) \
-    $(get_image_helm_args "${CLUSTER_NAME}" "${CLP_PACKAGE_IMAGE}")
+    $(get_image_helm_args "${CLUSTER_NAME}" "clpPackage" "${CLP_PACKAGE_IMAGE}")
 
 wait_for_cluster_ready

tools/deployment/package-helm/set-up-test.sh

@@ -30,6 +30,6 @@ sleep 2
 # shellcheck disable=SC2046
 helm install test "${script_dir}" \
     $(get_presto_helm_args) \
-    $(get_image_helm_args "${CLUSTER_NAME}" "${CLP_PACKAGE_IMAGE}")
+    $(get_image_helm_args "${CLUSTER_NAME}" "clpPackage" "${CLP_PACKAGE_IMAGE}")
 
 wait_for_cluster_ready

tools/deployment/package-helm/templates/_helpers.tpl

@@ -125,22 +125,30 @@ app.kubernetes.io/instance: {{ .Release.Name }}
 {{- end }}
 
 {{/*
-Creates image reference for the CLP Package.
+Creates a container image reference from .Values.image.
 
-@return {string} Full image reference (repository:tag)
-*/}}
-{{- define "clp.image.ref" -}}
-{{- $tag := .Values.image.clpPackage.tag | default .Chart.AppVersion }}
-{{- printf "%s:%s" .Values.image.clpPackage.repository $tag }}
-{{- end }}
-
-{{/*
-Creates image reference for the kubectl image.
+Renders repository@digest when "digest" is set; otherwise, renders repository:tag. clpPackage
+defaults to Chart.AppVersion when "tag" is omitted; other components require "tag".
 
-@return {string} Full image reference (repository@digest)
+@param {object} root Root template context (required)
+@param {string} component Key under .Values.image (e.g., "clpPackage", "redis")
+@return {string} Full image reference (repository@digest or repository:tag)
 */}}
-{{- define "clp.kubectl.image.ref" -}}
-{{- printf "%s@%s" .Values.image.kubectl.repository .Values.image.kubectl.digest }}
+{{- define "clp.imageRef" -}}
+{{- $img := index .root.Values.image .component -}}
+{{- if $img.digest -}}
+{{- printf "%s@%s" $img.repository $img.digest -}}
+{{- else -}}
+{{- $tag := $img.tag -}}
+{{- if not $tag -}}
+  {{- if eq .component "clpPackage" -}}
+    {{- $tag = .root.Chart.AppVersion -}}
+  {{- else -}}
+    {{- fail (printf "image.%s.tag is required" .component) -}}
+  {{- end -}}
+{{- end -}}
+{{- printf "%s:%s" $img.repository $tag -}}
+{{- end -}}
 {{- end }}
 
 {{/*
@@ -502,7 +510,8 @@ should be the job name suffix.
 */}}
 {{- define "clp.waitFor" -}}
 name: "wait-for-{{ .name }}"
-image: {{ include "clp.kubectl.image.ref" .root | quote }}
+image: {{ include "clp.imageRef" (dict "root" .root "component" "kubectl") | quote }}
+imagePullPolicy: {{ .root.Values.image.kubectl.pullPolicy | quote }}
 command: [
   "kubectl", "wait",
   {{- if eq .type "service" }}

tools/deployment/package-helm/templates/api-server-deployment.yaml

@@ -33,7 +33,7 @@ spec:
         - {{- include "clp.waitForResultsCache" . | nindent 10 }}
       containers:
         - name: "api-server"
-          image: "{{ include "clp.image.ref" . }}"
+          image: {{ include "clp.imageRef" (dict "root" . "component" "clpPackage") | quote }}
           imagePullPolicy: "{{ .Values.image.clpPackage.pullPolicy }}"
           env:
             {{- include "clp.telemetryEnv" . | nindent 12 }}

tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml

@@ -45,7 +45,7 @@ spec:
         {{- end }}
       containers:
         - name: "compression-scheduler"
-          image: "{{ include "clp.image.ref" . }}"
+          image: {{ include "clp.imageRef" (dict "root" . "component" "clpPackage") | quote }}
           imagePullPolicy: "{{ .Values.image.clpPackage.pullPolicy }}"
           env:
             - {{- include "clp.celeryBrokerUrlEnvVar" . | nindent 14 }}

tools/deployment/package-helm/templates/compression-worker-deployment.yaml

@@ -25,7 +25,7 @@ spec:
       terminationGracePeriodSeconds: 60
       containers:
         - name: "compression-worker"
-          image: "{{ include "clp.image.ref" . }}"
+          image: {{ include "clp.imageRef" (dict "root" . "component" "clpPackage") | quote }}
           imagePullPolicy: "{{ .Values.image.clpPackage.pullPolicy }}"
           env:
             - {{- include "clp.celeryBrokerUrlEnvVar" . | nindent 14 }}

tools/deployment/package-helm/templates/database-statefulset.yaml

@@ -25,13 +25,8 @@ spec:
         ) | nindent 6 }}
       containers:
         - name: "database"
-          image: >-
-            {{- if eq .Values.clpConfig.database.type "mysql" }}
-              mysql:8.0.46
-            {{- else }}
-              mariadb:10.11.16
-            {{- end }}
-          imagePullPolicy: "Always"
+          image: {{ include "clp.imageRef" (dict "root" . "component" .Values.clpConfig.database.type) | quote }}
+          imagePullPolicy: {{ (index .Values.image .Values.clpConfig.database.type).pullPolicy | quote }}
           env:
             - name: "MYSQL_DATABASE"
               value: {{ .Values.clpConfig.database.names.clp | quote }}

tools/deployment/package-helm/templates/db-table-creator-job.yaml

@@ -24,7 +24,7 @@ spec:
       {{- end }}
       containers:
         - name: "db-table-creator"
-          image: "{{ include "clp.image.ref" . }}"
+          image: {{ include "clp.imageRef" (dict "root" . "component" "clpPackage") | quote }}
           imagePullPolicy: "{{ .Values.image.clpPackage.pullPolicy }}"
           env:
             - name: "CLP_DB_PASS"

tools/deployment/package-helm/templates/garbage-collector-deployment.yaml

@@ -35,7 +35,7 @@ spec:
         - {{- include "clp.waitForResultsCache" . | nindent 10 }}
       containers:
         - name: "garbage-collector"
-          image: "{{ include "clp.image.ref" . }}"
+          image: {{ include "clp.imageRef" (dict "root" . "component" "clpPackage") | quote }}
           imagePullPolicy: "{{ .Values.image.clpPackage.pullPolicy }}"
           env:
             - name: "CLP_DB_PASS"

tools/deployment/package-helm/templates/log-ingestor-deployment.yaml

@@ -34,7 +34,7 @@ spec:
           ) | nindent 10 }}
       containers:
         - name: "log-ingestor"
-          image: "{{ include "clp.image.ref" . }}"
+          image: {{ include "clp.imageRef" (dict "root" . "component" "clpPackage") | quote }}
           imagePullPolicy: "{{ .Values.image.clpPackage.pullPolicy }}"
           env:
             {{- include "clp.telemetryEnv" . | nindent 12 }}

tools/deployment/package-helm/templates/mcp-server-deployment.yaml

@@ -33,7 +33,7 @@ spec:
         - {{- include "clp.waitForResultsCache" . | nindent 10 }}
       containers:
         - name: "mcp-server"
-          image: "{{ include "clp.image.ref" . }}"
+          image: {{ include "clp.imageRef" (dict "root" . "component" "clpPackage") | quote }}
           imagePullPolicy: "{{ .Values.image.clpPackage.pullPolicy }}"
           env:
             - name: "CLP_DB_PASS"

tools/deployment/package-helm/templates/presto-worker-deployment.yaml

@@ -31,7 +31,8 @@ spec:
             "name" "presto-coordinator"
           ) | nindent 10 }}
         - name: "setup-configs"
-          image: {{ include "clp.kubectl.image.ref" . | quote }}
+          image: {{ include "clp.imageRef" (dict "root" . "component" "kubectl") | quote }}
+          imagePullPolicy: {{ .Values.image.kubectl.pullPolicy | quote }}
           command: ["/bin/sh", "/scripts/presto-worker-setup-configs.sh"]
           volumeMounts:
             - name: "presto-config"

tools/deployment/package-helm/templates/query-scheduler-deployment.yaml

@@ -48,7 +48,7 @@ spec:
         {{- end }}
       containers:
         - name: "query-scheduler"
-          image: "{{ include "clp.image.ref" . }}"
+          image: {{ include "clp.imageRef" (dict "root" . "component" "clpPackage") | quote }}
           imagePullPolicy: "{{ .Values.image.clpPackage.pullPolicy }}"
           env:
             - {{- include "clp.celeryBrokerUrlEnvVar" . | nindent 14 }}

tools/deployment/package-helm/templates/query-worker-deployment.yaml

@@ -26,7 +26,7 @@ spec:
       terminationGracePeriodSeconds: 60
       containers:
         - name: "query-worker"
-          image: "{{ include "clp.image.ref" . }}"
+          image: {{ include "clp.imageRef" (dict "root" . "component" "clpPackage") | quote }}
           imagePullPolicy: "{{ .Values.image.clpPackage.pullPolicy }}"
           env:
             - {{- include "clp.celeryBrokerUrlEnvVar" . | nindent 14 }}

tools/deployment/package-helm/templates/queue-statefulset.yaml

@@ -25,8 +25,8 @@ spec:
         ) | nindent 6 }}
       containers:
         - name: "queue"
-          image: "rabbitmq:4.2.6"
-          imagePullPolicy: "Always"
+          image: {{ include "clp.imageRef" (dict "root" . "component" "queue") | quote }}
+          imagePullPolicy: {{ .Values.image.queue.pullPolicy | quote }}
           env:
             - name: "RABBITMQ_DEFAULT_USER"
               valueFrom:

tools/deployment/package-helm/templates/redis-statefulset.yaml

@@ -25,8 +25,8 @@ spec:
         ) | nindent 6 }}
       containers:
         - name: "redis"
-          image: "redis:7.4.8"
-          imagePullPolicy: "Always"
+          image: {{ include "clp.imageRef" (dict "root" . "component" "redis") | quote }}
+          imagePullPolicy: {{ .Values.image.redis.pullPolicy | quote }}
           env:
             - name: "REDIS_PASSWORD"
               valueFrom:

tools/deployment/package-helm/templates/reducer-deployment.yaml

@@ -35,7 +35,7 @@ spec:
         - {{- include "clp.waitForResultsCache" . | nindent 10 }}
       containers:
         - name: "reducer"
-          image: "{{ include "clp.image.ref" . }}"
+          image: {{ include "clp.imageRef" (dict "root" . "component" "clpPackage") | quote }}
           imagePullPolicy: "{{ .Values.image.clpPackage.pullPolicy }}"
           env:
             - name: "CLP_HOME"

tools/deployment/package-helm/templates/results-cache-indices-creator-job.yaml

@@ -17,7 +17,7 @@ spec:
       restartPolicy: "OnFailure"
       containers:
         - name: "results-cache-indices-creator"
-          image: "{{ include "clp.image.ref" . }}"
+          image: {{ include "clp.imageRef" (dict "root" . "component" "clpPackage") | quote }}
           imagePullPolicy: "{{ .Values.image.clpPackage.pullPolicy }}"
           env:
             - name: "PYTHONPATH"

tools/deployment/package-helm/templates/results-cache-statefulset.yaml

@@ -25,8 +25,8 @@ spec:
         ) | nindent 6 }}
       containers:
         - name: "results-cache"
-          image: "mongo:8.0.21"
-          imagePullPolicy: "Always"
+          image: {{ include "clp.imageRef" (dict "root" . "component" "resultsCache") | quote }}
+          imagePullPolicy: {{ .Values.image.resultsCache.pullPolicy | quote }}
           ports:
             - name: "results-cache"
               containerPort: 27017
@@ -57,7 +57,7 @@ spec:
             {{- include "clp.livenessProbeTimings" . | nindent 12 }}
             exec: *results-cache-health-check
         - name: "results-cache-init"
-          image: "{{ include "clp.image.ref" . }}"
+          image: {{ include "clp.imageRef" (dict "root" . "component" "clpPackage") | quote }}
           imagePullPolicy: "{{ .Values.image.clpPackage.pullPolicy }}"
           env:
             - name: "PYTHONPATH"

tools/deployment/package-helm/templates/webui-deployment.yaml

@@ -32,7 +32,7 @@ spec:
         - {{- include "clp.waitForResultsCache" . | nindent 10 }}
       containers:
         - name: "webui"
-          image: "{{ include "clp.image.ref" . }}"
+          image: {{ include "clp.imageRef" (dict "root" . "component" "clpPackage") | quote }}
           imagePullPolicy: "{{ .Values.image.clpPackage.pullPolicy }}"
           env:
             - name: "CLP_DB_PASS"

tools/deployment/package-helm/values.yaml

@@ -9,6 +9,14 @@ image:
     repository: "ghcr.io/y-scope/clp/clp-package"
     pullPolicy: "Always"
     tag: "main"
+  mariadb:
+    repository: "mariadb"
+    pullPolicy: "Always"
+    tag: "10.11.16"
+  mysql:
+    repository: "mysql"
+    pullPolicy: "Always"
+    tag: "8.0.46"
   curl:
     repository: "curlimages/curl"
     pullPolicy: "IfNotPresent"
@@ -30,6 +38,18 @@ image:
     repository: "ghcr.io/y-scope/presto/prestissimo-worker"
     pullPolicy: "IfNotPresent"
     tag: "clp-v0.10.0-fix.1"
+  queue:
+    repository: "rabbitmq"
+    pullPolicy: "Always"
+    tag: "4.2.6"
+  redis:
+    repository: "redis"
+    pullPolicy: "Always"
+    tag: "7.4.8"
+  resultsCache:
+    repository: "mongo"
+    pullPolicy: "Always"
+    tag: "8.0.21"
 
 # - If false: Single-node deployment.
 #   - Pods automatically tolerate control-plane taints.