From f459811d37452b5f93083879279fe5e345d76e66 Mon Sep 17 00:00:00 2001
From: Brian Newsom <brnewsom@nvidia.com>
Date: Thu, 30 Jul 2026 12:52:20 -0600
Subject: [PATCH 1/6] feat(intake): add intake clickhouse to nemo-platform helm

Signed-off-by: Brian Newsom <brnewsom@nvidia.com>
---
 k8s/helm/README.md                            | 114 ++++++++++++++++
 k8s/helm/ci/23-embedded-clickhouse.yaml       |  11 ++
 k8s/helm/ci/24-external-clickhouse.yaml       |  11 ++
 .../nemo-helm-readme.md.gotmpl                |  76 +++++++++++
 k8s/helm/templates/NOTES.txt                  |  13 ++
 k8s/helm/templates/_helpers.tpl               |  84 ++++++++++++
 k8s/helm/templates/api/api-deployment.yaml    |  11 ++
 .../clickhouse/clickhouse-secret.yaml         |  12 ++
 .../clickhouse/clickhouse-service.yaml        |  27 ++++
 .../clickhouse/clickhouse-serviceaccount.yaml |  14 ++
 .../clickhouse/clickhouse-statefulset.yaml    | 126 ++++++++++++++++++
 k8s/helm/values.yaml                          |  91 +++++++++++++
 tools/lint/lint-helm.sh                       |  18 +++
 13 files changed, 608 insertions(+)
 create mode 100644 k8s/helm/ci/23-embedded-clickhouse.yaml
 create mode 100644 k8s/helm/ci/24-external-clickhouse.yaml
 create mode 100644 k8s/helm/templates/clickhouse/clickhouse-secret.yaml
 create mode 100644 k8s/helm/templates/clickhouse/clickhouse-service.yaml
 create mode 100644 k8s/helm/templates/clickhouse/clickhouse-serviceaccount.yaml
 create mode 100644 k8s/helm/templates/clickhouse/clickhouse-statefulset.yaml

diff --git a/k8s/helm/README.md b/k8s/helm/README.md
index 36e6d8e434..ce5b3cfb01 100644
--- a/k8s/helm/README.md
+++ b/k8s/helm/README.md
@@ -29,6 +29,82 @@ On upgrade, the generated Secret must already exist and contain
 Secret instead of generating a replacement key; existing encrypted platform
 secrets will not decrypt with a new key.
 
+## Intake and ClickHouse
+
+Intake is included in the platform API service group. By default, the chart
+deploys a single-node embedded ClickHouse 26.3 LTS service, and Intake creates
+and migrates its own `intake` database on first use.
+
+The embedded ClickHouse is intended for development, evaluation, and
+non-critical single-node installations. It does not provide replication or
+automatic backups. For a production deployment that requires high availability,
+set `clickhouse.enabled` to `false` and provide a separately managed ClickHouse:
+
+```yaml
+clickhouse:
+  enabled: false
+
+externalClickhouse:
+  host: clickhouse.example.internal
+  port: 8443
+  secure: true
+  user: nemo
+  database: intake
+  existingSecret: clickhouse-credentials
+  existingSecretPasswordKey: password
+```
+
+The external user must be allowed to create the configured database, tables,
+materialized views, and indexes because Intake owns its ClickHouse migrations.
+
+### ClickHouse sizing
+
+Use retained span count as an initial operational threshold, not as a disk-size
+estimate. Span `input`, `output`, and attribute payloads vary substantially.
+Measure `bytes_on_disk` from representative traffic before setting production
+storage:
+
+```sql
+SELECT
+    table,
+    sum(rows) AS physical_rows,
+    formatReadableSize(sum(bytes_on_disk)) AS disk
+FROM system.parts
+WHERE active AND database = 'intake'
+GROUP BY table
+ORDER BY table;
+```
+
+The following are starting points for the current Intake schema and interactive
+query workload:
+
+| Retained Intake spans | Topology | ClickHouse resources | Storage |
+| --- | --- | --- | --- |
+| Up to 1 million | Embedded single node for non-critical workloads | 2 vCPU, 8 GiB RAM | Fast SSD, at least 20 GiB and 2× measured active data |
+| 1–10 million | External preferred; embedded only when downtime and data loss are acceptable | 4–8 vCPU, 16–32 GiB RAM | Provisioned-IOPS SSD, at least 100 GiB and 2× measured active data |
+| More than 10 million, or any HA requirement | Managed ClickHouse or an operator-managed replicated cluster | Start at 8 vCPU and 32 GiB RAM per replica, then load-test the actual ingest/read mix | Size from measured compression, retention, replication, and merge headroom |
+
+Intake currently retains spans and its trace index for 90 days. Evaluator
+results and annotations do not have a time-based TTL, so include their continuing
+growth in capacity planning. ReplacingMergeTree retries also leave physical row
+versions until background merges complete.
+
+For large or frequently queried deployments, ClickHouse recommends:
+
+- At least 8 GiB RAM even at low data volumes.
+- A general-purpose starting ratio of 4 GiB RAM per CPU core.
+- Provisioned-IOPS SSDs for latency-sensitive workloads.
+- Roughly 1:30 to 1:50 RAM-to-storage for frequently accessed large datasets.
+- Replication for production durability; vertically scale replicas before
+  adding shards.
+
+Validate CPU, query peak memory, active parts, merge backlog, disk latency, and
+free space under representative batched OTLP ingestion before promoting a tier.
+See the upstream
+[ClickHouse sizing guide](https://clickhouse.com/docs/guides/oss/best-practices/sizing-and-hardware-recommendations)
+and
+[OSS operational recommendations](https://clickhouse.com/docs/guides/oss/best-practices/tips).
+
 ## Values
 
 | Key | Type | Default | Description |
@@ -97,6 +173,36 @@ secrets will not decrypt with a new key.
 | api.tolerations | list | `[]` | Tolerations configuration for the API service. |
 | api.topologySpreadConstraints | list | `[]` | Topology spread constraints for the API service pods. See https://kubernetes.io/docs/concepts/scheduling-eviction/topology-spread-constraints/ |
 | basePlatformConfig | string | This object has the following default values for the base platform configuration. | Base platform configuration settings |
+| clickhouse | object | This object has the following default values for the embedded ClickHouse configuration. | Embedded ClickHouse configuration for Intake. The embedded deployment is a single-node convenience topology. Use an external, replicated ClickHouse deployment for production environments that require high availability. These values are used only when `clickhouse.enabled` is true. |
+| clickhouse.affinity | object | `{}` | Affinity for the ClickHouse pod. |
+| clickhouse.annotations | object | `{}` | Annotations to add to the ClickHouse StatefulSet. |
+| clickhouse.auth.database | string | `"intake"` | ClickHouse database used by Intake. |
+| clickhouse.auth.existingSecret | string | `""` | Name of an existing Secret containing the ClickHouse password. If empty, the chart creates one. |
+| clickhouse.auth.existingSecretPasswordKey | string | `"password"` | Key in auth.existingSecret containing the ClickHouse password. |
+| clickhouse.auth.password | string | `"nemo"` | ClickHouse password used when auth.existingSecret is empty. |
+| clickhouse.auth.username | string | `"nemo"` | ClickHouse username used by Intake. |
+| clickhouse.enabled | bool | `true` | Whether to deploy the embedded ClickHouse. Set to false to use `externalClickhouse`. |
+| clickhouse.image.pullPolicy | string | `"IfNotPresent"` | ClickHouse image pull policy. |
+| clickhouse.image.repository | string | `"docker.io/clickhouse/clickhouse-server"` | ClickHouse image repository. Intake is tested against the 26.3 LTS release line. |
+| clickhouse.image.tag | string | `"26.3"` | ClickHouse image tag. |
+| clickhouse.nodeSelector | object | `{}` | Node selector for the ClickHouse pod. |
+| clickhouse.persistence.enabled | bool | `true` | Whether to persist embedded ClickHouse data. |
+| clickhouse.persistence.size | string | `"20Gi"` | PersistentVolumeClaim size. See the Intake and ClickHouse sizing guidance in the chart README. |
+| clickhouse.persistence.storageClass | string | `""` | Storage class for the ClickHouse PVC. If unset, the cluster default is used. |
+| clickhouse.podAnnotations | object | `{}` | Annotations to add to the ClickHouse pod. |
+| clickhouse.podLabels | object | `{}` | Additional labels to add to the ClickHouse pod. |
+| clickhouse.podSecurityContext | object | `{}` | Optional pod security context for the ClickHouse pod. |
+| clickhouse.resources | object | `{"requests":{"cpu":"2","memory":"8Gi"}}` | Resource requests and limits for the ClickHouse container. The defaults are the supported small-volume starting point. |
+| clickhouse.securityContext | object | `{}` | Optional container security context for the ClickHouse container. |
+| clickhouse.service.annotations | object | `{}` | Annotations to add to the ClickHouse Service. |
+| clickhouse.service.httpPort | int | `8123` | ClickHouse HTTP interface port used by Intake. |
+| clickhouse.service.nativePort | int | `9000` | ClickHouse native protocol port exposed inside the cluster for administration. |
+| clickhouse.serviceAccount | object | This object has the following default values for the service account configuration. | Service account for the ClickHouse pod. |
+| clickhouse.serviceAccount.annotations | object | `{}` | Annotations to add to the service account. |
+| clickhouse.serviceAccount.automount | bool | `false` | Automatically mount the ServiceAccount's API credentials. |
+| clickhouse.serviceAccount.create | bool | `true` | Specifies whether a service account should be created for the ClickHouse pod. |
+| clickhouse.serviceAccount.name | string | `""` | The name of the service account to use. If not set and create is true, a name is generated from the release fullname. |
+| clickhouse.tolerations | list | `[]` | Tolerations for the ClickHouse pod. |
 | core | object | This object has the following default values for the core deployment configuration. | Core deployment configuration settings |
 | core.controller.affinity | object | `{}` | Affinity configuration for the controller service. |
 | core.controller.annotations | object | `{}` | Annotations to add to the controller service deployment. |
@@ -214,6 +320,14 @@ secrets will not decrypt with a new key.
 | envoyProxy.tolerations | list | `[]` | Tolerations configuration for the Envoy pods. |
 | envoyProxy.topologySpreadConstraints | list | `[]` | Topology spread constraints for the Envoy pods. See https://kubernetes.io/docs/concepts/scheduling-eviction/topology-spread-constraints/ |
 | existingSecret | string | `"ngc-api"` | You can use an existing Kubernetes secret for communicating with the NGC API for downloading models. The chart uses the `ngcAPIKey` value to generate the secret if you set this to an empty string. |
+| externalClickhouse | object | This object has the following default values for the external ClickHouse configuration. | External ClickHouse configuration. These values are used when `clickhouse.enabled` is false. |
+| externalClickhouse.database | string | `"intake"` | ClickHouse database used by Intake. |
+| externalClickhouse.existingSecret | string | `""` | Name of an existing Secret containing the ClickHouse password. Required when using external ClickHouse. |
+| externalClickhouse.existingSecretPasswordKey | string | `""` | Key in existingSecret containing the ClickHouse password. Required when using external ClickHouse. |
+| externalClickhouse.host | string | `""` | External ClickHouse host. Required when the embedded ClickHouse is disabled. |
+| externalClickhouse.port | int | `8123` | External ClickHouse HTTP interface port. |
+| externalClickhouse.secure | bool | `false` | Whether Intake should connect to ClickHouse over HTTPS. |
+| externalClickhouse.user | string | `"nemo"` | ClickHouse username used by Intake. |
 | externalDatabase | object | This object has the following default values for the external PostgreSQL configuration. | External PostgreSQL configuration settings. These values are only used when postgresql.enabled is set to false. |
 | externalDatabase.database | string | `"nemoplatform"` | Database name. |
 | externalDatabase.existingSecret | string | `""` | Name of an existing secret resource containing the database credentials. |
diff --git a/k8s/helm/ci/23-embedded-clickhouse.yaml b/k8s/helm/ci/23-embedded-clickhouse.yaml
new file mode 100644
index 0000000000..bd6a3aca20
--- /dev/null
+++ b/k8s/helm/ci/23-embedded-clickhouse.yaml
@@ -0,0 +1,11 @@
+clickhouse:
+  persistence:
+    size: 50Gi
+    storageClass: fast-ssd
+  podAnnotations:
+    example.com/role: telemetry-store
+  podLabels:
+    workload: intake
+  service:
+    annotations:
+      example.com/service: clickhouse
diff --git a/k8s/helm/ci/24-external-clickhouse.yaml b/k8s/helm/ci/24-external-clickhouse.yaml
new file mode 100644
index 0000000000..7711a600a6
--- /dev/null
+++ b/k8s/helm/ci/24-external-clickhouse.yaml
@@ -0,0 +1,11 @@
+clickhouse:
+  enabled: false
+
+externalClickhouse:
+  host: clickhouse.example.internal
+  port: 8443
+  secure: true
+  user: intake
+  database: intake
+  existingSecret: clickhouse-credentials
+  existingSecretPasswordKey: clickhouse-password
diff --git a/k8s/helm/helm-docs-template/nemo-helm-readme.md.gotmpl b/k8s/helm/helm-docs-template/nemo-helm-readme.md.gotmpl
index a4cccd775a..93c0061ade 100644
--- a/k8s/helm/helm-docs-template/nemo-helm-readme.md.gotmpl
+++ b/k8s/helm/helm-docs-template/nemo-helm-readme.md.gotmpl
@@ -29,4 +29,80 @@ On upgrade, the generated Secret must already exist and contain
 Secret instead of generating a replacement key; existing encrypted platform
 secrets will not decrypt with a new key.
 
+## Intake and ClickHouse
+
+Intake is included in the platform API service group. By default, the chart
+deploys a single-node embedded ClickHouse 26.3 LTS service, and Intake creates
+and migrates its own `intake` database on first use.
+
+The embedded ClickHouse is intended for development, evaluation, and
+non-critical single-node installations. It does not provide replication or
+automatic backups. For a production deployment that requires high availability,
+set `clickhouse.enabled` to `false` and provide a separately managed ClickHouse:
+
+```yaml
+clickhouse:
+  enabled: false
+
+externalClickhouse:
+  host: clickhouse.example.internal
+  port: 8443
+  secure: true
+  user: nemo
+  database: intake
+  existingSecret: clickhouse-credentials
+  existingSecretPasswordKey: password
+```
+
+The external user must be allowed to create the configured database, tables,
+materialized views, and indexes because Intake owns its ClickHouse migrations.
+
+### ClickHouse sizing
+
+Use retained span count as an initial operational threshold, not as a disk-size
+estimate. Span `input`, `output`, and attribute payloads vary substantially.
+Measure `bytes_on_disk` from representative traffic before setting production
+storage:
+
+```sql
+SELECT
+    table,
+    sum(rows) AS physical_rows,
+    formatReadableSize(sum(bytes_on_disk)) AS disk
+FROM system.parts
+WHERE active AND database = 'intake'
+GROUP BY table
+ORDER BY table;
+```
+
+The following are starting points for the current Intake schema and interactive
+query workload:
+
+| Retained Intake spans | Topology | ClickHouse resources | Storage |
+| --- | --- | --- | --- |
+| Up to 1 million | Embedded single node for non-critical workloads | 2 vCPU, 8 GiB RAM | Fast SSD, at least 20 GiB and 2× measured active data |
+| 1–10 million | External preferred; embedded only when downtime and data loss are acceptable | 4–8 vCPU, 16–32 GiB RAM | Provisioned-IOPS SSD, at least 100 GiB and 2× measured active data |
+| More than 10 million, or any HA requirement | Managed ClickHouse or an operator-managed replicated cluster | Start at 8 vCPU and 32 GiB RAM per replica, then load-test the actual ingest/read mix | Size from measured compression, retention, replication, and merge headroom |
+
+Intake currently retains spans and its trace index for 90 days. Evaluator
+results and annotations do not have a time-based TTL, so include their continuing
+growth in capacity planning. ReplacingMergeTree retries also leave physical row
+versions until background merges complete.
+
+For large or frequently queried deployments, ClickHouse recommends:
+
+- At least 8 GiB RAM even at low data volumes.
+- A general-purpose starting ratio of 4 GiB RAM per CPU core.
+- Provisioned-IOPS SSDs for latency-sensitive workloads.
+- Roughly 1:30 to 1:50 RAM-to-storage for frequently accessed large datasets.
+- Replication for production durability; vertically scale replicas before
+  adding shards.
+
+Validate CPU, query peak memory, active parts, merge backlog, disk latency, and
+free space under representative batched OTLP ingestion before promoting a tier.
+See the upstream
+[ClickHouse sizing guide](https://clickhouse.com/docs/guides/oss/best-practices/sizing-and-hardware-recommendations)
+and
+[OSS operational recommendations](https://clickhouse.com/docs/guides/oss/best-practices/tips).
+
 {{ template "chart.valuesSection" . }}
diff --git a/k8s/helm/templates/NOTES.txt b/k8s/helm/templates/NOTES.txt
index c604165176..d81aa4c482 100644
--- a/k8s/helm/templates/NOTES.txt
+++ b/k8s/helm/templates/NOTES.txt
@@ -95,6 +95,19 @@ Successfully installed {{ .Chart.Name }}-{{ .Chart.Version }}, named {{ .Release
 {{- end }}
 {{- end }}
 
+  Intake is enabled.
+{{- if .Values.clickhouse.enabled }}
+  The embedded single-node ClickHouse is intended for non-critical workloads.
+  Use an external replicated ClickHouse for production high availability.
+{{- if not .Values.clickhouse.persistence.enabled }}
+
+  CLICKHOUSE PERSISTENCE DISABLED. Intake telemetry will be lost if the pod restarts.
+{{- end }}
+{{- else }}
+  Intake is configured to use external ClickHouse at
+  {{ include "nemo-common.clickhouse.url" . }}.
+{{- end }}
+
 {{- if or .Values.multinodeNetworking.aws.enabled .Values.multinodeNetworking.azure.enabled .Values.multinodeNetworking.gcp.enabled .Values.multinodeNetworking.oci.enabled }}
 {{- $ncclFullname := include "nemo-platform.fullname" . }}
 {{- $ncclTestBase := printf "%s-nccl-test" ($ncclFullname | trunc 40 | trimSuffix "-") | trunc 42 }}
diff --git a/k8s/helm/templates/_helpers.tpl b/k8s/helm/templates/_helpers.tpl
index cb4be28521..60ecd15110 100644
--- a/k8s/helm/templates/_helpers.tpl
+++ b/k8s/helm/templates/_helpers.tpl
@@ -342,6 +342,90 @@ nemo-common.database.password generates a POSTGRES_DB_PASSWORD environment value
 {{- end }}
 {{- end -}}
 
+{{/*
+Embedded ClickHouse full name (service and generated secret name).
+*/}}
+{{- define "nemo-common.clickhouse.fullname" -}}
+{{- printf "%s-clickhouse" (include "nemo-platform.fullname" . | trunc 51 | trimSuffix "-") -}}
+{{- end -}}
+
+{{/*
+Name of the service account to use for the embedded ClickHouse pod.
+*/}}
+{{- define "nemo-common.clickhouse.serviceAccountName" -}}
+{{- if .Values.clickhouse.serviceAccount.create -}}
+{{- default (include "nemo-common.clickhouse.fullname" .) .Values.clickhouse.serviceAccount.name -}}
+{{- else -}}
+{{- default "default" .Values.clickhouse.serviceAccount.name -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Whether the embedded ClickHouse should be rendered.
+*/}}
+{{- define "nemo-common.clickhouse.enabled" -}}
+{{- if .Values.clickhouse.enabled -}}
+true
+{{- end -}}
+{{- end -}}
+
+{{/*
+ClickHouse HTTP URL used by Intake.
+*/}}
+{{- define "nemo-common.clickhouse.url" -}}
+{{- if .Values.clickhouse.enabled -}}
+{{- printf "http://%s:%d" (include "nemo-common.clickhouse.fullname" .) (.Values.clickhouse.service.httpPort | int) -}}
+{{- else -}}
+{{- $host := required "externalClickhouse.host is required when clickhouse.enabled=false" .Values.externalClickhouse.host -}}
+{{- $scheme := ternary "https" "http" .Values.externalClickhouse.secure -}}
+{{- printf "%s://%s:%d" $scheme $host (.Values.externalClickhouse.port | int) -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+ClickHouse username used by Intake.
+*/}}
+{{- define "nemo-common.clickhouse.user" -}}
+{{- if .Values.clickhouse.enabled -}}
+{{- .Values.clickhouse.auth.username -}}
+{{- else -}}
+{{- .Values.externalClickhouse.user -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+ClickHouse database used by Intake.
+*/}}
+{{- define "nemo-common.clickhouse.database" -}}
+{{- if .Values.clickhouse.enabled -}}
+{{- .Values.clickhouse.auth.database -}}
+{{- else -}}
+{{- .Values.externalClickhouse.database -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Secret containing the ClickHouse password used by Intake.
+*/}}
+{{- define "nemo-common.clickhouse.secretName" -}}
+{{- if .Values.clickhouse.enabled -}}
+{{- default (include "nemo-common.clickhouse.fullname" .) .Values.clickhouse.auth.existingSecret -}}
+{{- else -}}
+{{- required "externalClickhouse.existingSecret is required when clickhouse.enabled=false" .Values.externalClickhouse.existingSecret -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Key containing the ClickHouse password used by Intake.
+*/}}
+{{- define "nemo-common.clickhouse.passwordKey" -}}
+{{- if .Values.clickhouse.enabled -}}
+{{- .Values.clickhouse.auth.existingSecretPasswordKey | default "password" -}}
+{{- else -}}
+{{- required "externalClickhouse.existingSecretPasswordKey is required when clickhouse.enabled=false" .Values.externalClickhouse.existingSecretPasswordKey -}}
+{{- end -}}
+{{- end -}}
+
 {{/*
 nemo-common.otel-env generates an env var array from the top-level telemetry configuration.
 Follows the specification at https://opentelemetry.io/docs/specs/otel/configuration/sdk-environment-variables/
diff --git a/k8s/helm/templates/api/api-deployment.yaml b/k8s/helm/templates/api/api-deployment.yaml
index 5a95170bad..f24659de37 100644
--- a/k8s/helm/templates/api/api-deployment.yaml
+++ b/k8s/helm/templates/api/api-deployment.yaml
@@ -69,6 +69,17 @@ spec:
             {{- end }}
             - name: NMP_CONFIG_FILE_PATH
               value: /etc/nmp/config.yaml
+            - name: NMP_INTAKE_CLICKHOUSE_URL
+              value: {{ include "nemo-common.clickhouse.url" . | quote }}
+            - name: NMP_INTAKE_CLICKHOUSE_USER
+              value: {{ include "nemo-common.clickhouse.user" . | quote }}
+            - name: NMP_INTAKE_CLICKHOUSE_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: {{ include "nemo-common.clickhouse.secretName" . | quote }}
+                  key: {{ include "nemo-common.clickhouse.passwordKey" . | quote }}
+            - name: NMP_INTAKE_CLICKHOUSE_DATABASE
+              value: {{ include "nemo-common.clickhouse.database" . | quote }}
             - name: NMP_BASE_URL
               {{- if include "nemo-platform.embeddedPdpEnabled" . }}
               value: {{ include "nemo-platform.apiLoopbackBaseUrl" . | quote }}
diff --git a/k8s/helm/templates/clickhouse/clickhouse-secret.yaml b/k8s/helm/templates/clickhouse/clickhouse-secret.yaml
new file mode 100644
index 0000000000..7bc42fc9e8
--- /dev/null
+++ b/k8s/helm/templates/clickhouse/clickhouse-secret.yaml
@@ -0,0 +1,12 @@
+{{- if and .Values.clickhouse.enabled (not .Values.clickhouse.auth.existingSecret) }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ include "nemo-common.clickhouse.fullname" . }}
+  labels:
+    app.kubernetes.io/component: clickhouse
+    {{- include "nemo-platform.labels" . | nindent 4 }}
+type: Opaque
+stringData:
+  password: {{ .Values.clickhouse.auth.password | quote }}
+{{- end }}
diff --git a/k8s/helm/templates/clickhouse/clickhouse-service.yaml b/k8s/helm/templates/clickhouse/clickhouse-service.yaml
new file mode 100644
index 0000000000..228942ffa3
--- /dev/null
+++ b/k8s/helm/templates/clickhouse/clickhouse-service.yaml
@@ -0,0 +1,27 @@
+{{- if include "nemo-common.clickhouse.enabled" . }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "nemo-common.clickhouse.fullname" . }}
+  labels:
+    app.kubernetes.io/component: clickhouse
+    {{- include "nemo-platform.labels" . | nindent 4 }}
+  {{- with .Values.clickhouse.service.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+spec:
+  type: ClusterIP
+  ports:
+    - name: http
+      port: {{ .Values.clickhouse.service.httpPort | int }}
+      targetPort: http
+      protocol: TCP
+    - name: native
+      port: {{ .Values.clickhouse.service.nativePort | int }}
+      targetPort: native
+      protocol: TCP
+  selector:
+    app.kubernetes.io/component: clickhouse
+    {{- include "nemo-platform.selectorLabels" . | nindent 4 }}
+{{- end }}
diff --git a/k8s/helm/templates/clickhouse/clickhouse-serviceaccount.yaml b/k8s/helm/templates/clickhouse/clickhouse-serviceaccount.yaml
new file mode 100644
index 0000000000..6f975b8aae
--- /dev/null
+++ b/k8s/helm/templates/clickhouse/clickhouse-serviceaccount.yaml
@@ -0,0 +1,14 @@
+{{- if and (include "nemo-common.clickhouse.enabled" .) .Values.clickhouse.serviceAccount.create }}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ include "nemo-common.clickhouse.serviceAccountName" . }}
+  labels:
+    app.kubernetes.io/component: clickhouse
+    {{- include "nemo-platform.labels" . | nindent 4 }}
+  {{- with .Values.clickhouse.serviceAccount.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+automountServiceAccountToken: {{ .Values.clickhouse.serviceAccount.automount }}
+{{- end }}
diff --git a/k8s/helm/templates/clickhouse/clickhouse-statefulset.yaml b/k8s/helm/templates/clickhouse/clickhouse-statefulset.yaml
new file mode 100644
index 0000000000..7011c1c058
--- /dev/null
+++ b/k8s/helm/templates/clickhouse/clickhouse-statefulset.yaml
@@ -0,0 +1,126 @@
+{{- if include "nemo-common.clickhouse.enabled" . }}
+{{- $imagePullSecrets := include "nemo-common.imagepullsecrets" . }}
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: {{ include "nemo-common.clickhouse.fullname" . }}
+  labels:
+    app.kubernetes.io/component: clickhouse
+    {{- include "nemo-platform.labels" . | nindent 4 }}
+  {{- with .Values.clickhouse.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+spec:
+  serviceName: {{ include "nemo-common.clickhouse.fullname" . }}
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/component: clickhouse
+      {{- include "nemo-platform.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      {{- with .Values.clickhouse.podAnnotations }}
+      annotations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      labels:
+        app.kubernetes.io/component: clickhouse
+        {{- include "nemo-platform.labels" . | nindent 8 }}
+        {{- with .Values.clickhouse.podLabels }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+    spec:
+      serviceAccountName: {{ include "nemo-common.clickhouse.serviceAccountName" . }}
+{{- if $imagePullSecrets }}
+      imagePullSecrets:
+{{- $imagePullSecrets | nindent 8 }}
+{{- end }}
+      securityContext:
+        {{- include "nemo-common.podSecurityContext" (dict "global" .Values.podSecurityContext "local" .Values.clickhouse.podSecurityContext) | nindent 8 }}
+      {{- with .Values.clickhouse.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.clickhouse.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.clickhouse.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      containers:
+        - name: clickhouse
+          image: {{ printf "%s:%s" .Values.clickhouse.image.repository .Values.clickhouse.image.tag | quote }}
+          imagePullPolicy: {{ .Values.clickhouse.image.pullPolicy }}
+          securityContext:
+            {{- include "nemo-common.securityContext" (dict "global" .Values.securityContext "local" .Values.clickhouse.securityContext) | nindent 12 }}
+          env:
+            - name: CLICKHOUSE_USER
+              value: {{ .Values.clickhouse.auth.username | quote }}
+            - name: CLICKHOUSE_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: {{ include "nemo-common.clickhouse.secretName" . | quote }}
+                  key: {{ include "nemo-common.clickhouse.passwordKey" . | quote }}
+            - name: CLICKHOUSE_DB
+              value: {{ .Values.clickhouse.auth.database | quote }}
+            - name: CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT
+              value: "1"
+          ports:
+            - name: http
+              containerPort: 8123
+              protocol: TCP
+            - name: native
+              containerPort: 9000
+              protocol: TCP
+          startupProbe:
+            httpGet:
+              path: /ping
+              port: http
+            periodSeconds: 5
+            timeoutSeconds: 3
+            failureThreshold: 60
+          livenessProbe:
+            httpGet:
+              path: /ping
+              port: http
+            periodSeconds: 10
+            timeoutSeconds: 5
+            failureThreshold: 3
+          readinessProbe:
+            httpGet:
+              path: /ping
+              port: http
+            periodSeconds: 5
+            timeoutSeconds: 3
+            failureThreshold: 3
+          volumeMounts:
+            - name: data
+              mountPath: /var/lib/clickhouse
+          resources:
+            {{- toYaml .Values.clickhouse.resources | nindent 12 }}
+      {{- if not .Values.clickhouse.persistence.enabled }}
+      volumes:
+        - name: data
+          emptyDir: {}
+      {{- end }}
+  {{- if .Values.clickhouse.persistence.enabled }}
+  volumeClaimTemplates:
+    - metadata:
+        name: data
+        labels:
+          app.kubernetes.io/component: clickhouse
+          {{- include "nemo-platform.labels" . | nindent 10 }}
+      spec:
+        accessModes:
+          - ReadWriteOnce
+        resources:
+          requests:
+            storage: {{ .Values.clickhouse.persistence.size | quote }}
+        {{- with .Values.clickhouse.persistence.storageClass }}
+        storageClassName: {{ . | quote }}
+        {{- end }}
+  {{- end }}
+{{- end }}
diff --git a/k8s/helm/values.yaml b/k8s/helm/values.yaml
index 16043d60c1..e58b2f7615 100644
--- a/k8s/helm/values.yaml
+++ b/k8s/helm/values.yaml
@@ -242,6 +242,97 @@ externalDatabase:
     # -- Key in the URI secret containing the database URI.
     key: ""
 
+# -- Embedded ClickHouse configuration for Intake.
+# The embedded deployment is a single-node convenience topology. Use an external,
+# replicated ClickHouse deployment for production environments that require high availability.
+# These values are used only when `clickhouse.enabled` is true.
+# @default -- This object has the following default values for the embedded ClickHouse configuration.
+clickhouse:
+  # -- Whether to deploy the embedded ClickHouse. Set to false to use `externalClickhouse`.
+  enabled: true
+  image:
+    # -- ClickHouse image repository. Intake is tested against the 26.3 LTS release line.
+    repository: docker.io/clickhouse/clickhouse-server
+    # -- ClickHouse image tag.
+    tag: "26.3"
+    # -- ClickHouse image pull policy.
+    pullPolicy: IfNotPresent
+  auth:
+    # -- ClickHouse username used by Intake.
+    username: nemo
+    # -- ClickHouse password used when auth.existingSecret is empty.
+    password: nemo
+    # -- ClickHouse database used by Intake.
+    database: intake
+    # -- Name of an existing Secret containing the ClickHouse password. If empty, the chart creates one.
+    existingSecret: ""
+    # -- Key in auth.existingSecret containing the ClickHouse password.
+    existingSecretPasswordKey: password
+  service:
+    # -- ClickHouse HTTP interface port used by Intake.
+    httpPort: 8123
+    # -- ClickHouse native protocol port exposed inside the cluster for administration.
+    nativePort: 9000
+    # -- Annotations to add to the ClickHouse Service.
+    annotations: {}
+  persistence:
+    # -- Whether to persist embedded ClickHouse data.
+    enabled: true
+    # -- PersistentVolumeClaim size. See the Intake and ClickHouse sizing guidance in the chart README.
+    size: 20Gi
+    # -- Storage class for the ClickHouse PVC. If unset, the cluster default is used.
+    storageClass: ""
+  # -- Resource requests and limits for the ClickHouse container. The defaults are the supported small-volume starting point.
+  resources:
+    requests:
+      cpu: "2"
+      memory: 8Gi
+  # -- Optional pod security context for the ClickHouse pod.
+  podSecurityContext: {}
+  # -- Optional container security context for the ClickHouse container.
+  securityContext: {}
+  # -- Service account for the ClickHouse pod.
+  # @default -- This object has the following default values for the service account configuration.
+  serviceAccount:
+    # -- Specifies whether a service account should be created for the ClickHouse pod.
+    create: true
+    # -- Automatically mount the ServiceAccount's API credentials.
+    automount: false
+    # -- Annotations to add to the service account.
+    annotations: {}
+    # -- The name of the service account to use. If not set and create is true, a name is generated from the release fullname.
+    name: ""
+  # -- Annotations to add to the ClickHouse StatefulSet.
+  annotations: {}
+  # -- Annotations to add to the ClickHouse pod.
+  podAnnotations: {}
+  # -- Additional labels to add to the ClickHouse pod.
+  podLabels: {}
+  # -- Node selector for the ClickHouse pod.
+  nodeSelector: {}
+  # -- Affinity for the ClickHouse pod.
+  affinity: {}
+  # -- Tolerations for the ClickHouse pod.
+  tolerations: []
+
+# -- External ClickHouse configuration. These values are used when `clickhouse.enabled` is false.
+# @default -- This object has the following default values for the external ClickHouse configuration.
+externalClickhouse:
+  # -- External ClickHouse host. Required when the embedded ClickHouse is disabled.
+  host: ""
+  # -- External ClickHouse HTTP interface port.
+  port: 8123
+  # -- Whether Intake should connect to ClickHouse over HTTPS.
+  secure: false
+  # -- ClickHouse username used by Intake.
+  user: nemo
+  # -- ClickHouse database used by Intake.
+  database: intake
+  # -- Name of an existing Secret containing the ClickHouse password. Required when using external ClickHouse.
+  existingSecret: ""
+  # -- Key in existingSecret containing the ClickHouse password. Required when using external ClickHouse.
+  existingSecretPasswordKey: ""
+
 # -- Platform-wide configuration settings
 # Set configuration here to apply custom, structured configuration across all services.
 # Applied after the base platform config is evaluated for templates. Enables adding / overriding YAML-based elements in the evaluated platform config.
diff --git a/tools/lint/lint-helm.sh b/tools/lint/lint-helm.sh
index 2a3de08c72..cb6c3b7520 100755
--- a/tools/lint/lint-helm.sh
+++ b/tools/lint/lint-helm.sh
@@ -36,6 +36,24 @@ envoy_autoscaling_output=$(helm template "${HELM_RELEASE_NAME}" "${HELM_FOLDER}"
 grep -Fq "envoyProxy.resources.requests.cpu is required when Envoy CPU autoscaling is enabled" \
   <<<"${envoy_autoscaling_output}"
 
+# Intake must not render with an incomplete external ClickHouse connection.
+external_clickhouse_output=$(helm template "${HELM_RELEASE_NAME}" "${HELM_FOLDER}" \
+  --set clickhouse.enabled=false 2>&1) && {
+  echo "Intake accepted a missing external ClickHouse host" >&2
+  exit 1
+}
+grep -Fq "externalClickhouse.host is required when clickhouse.enabled=false" \
+  <<<"${external_clickhouse_output}"
+
+external_clickhouse_secret_output=$(helm template "${HELM_RELEASE_NAME}" "${HELM_FOLDER}" \
+  --set clickhouse.enabled=false \
+  --set externalClickhouse.host=clickhouse.example.internal 2>&1) && {
+  echo "External ClickHouse accepted a missing credentials Secret" >&2
+  exit 1
+}
+grep -Fq "externalClickhouse.existingSecret is required when clickhouse.enabled=false" \
+  <<<"${external_clickhouse_secret_output}"
+
 # Validate the Helm chart by rendering templates with all values files in ci/ directory
 shopt -s nullglob
 for value_file in "${HELM_FOLDER}"/ci/*.yaml; do

From 3ec7b56dd3e116f5be10c9f92eb71a7b85f374cf Mon Sep 17 00:00:00 2001
From: Brian Newsom <brnewsom@nvidia.com>
Date: Thu, 30 Jul 2026 13:14:16 -0600
Subject: [PATCH 2/6] fix(helm): harden clickhouse credentials

Signed-off-by: Brian Newsom <brnewsom@nvidia.com>
---
 k8s/helm/README.md                                 | 13 ++++++++++++-
 .../helm-docs-template/nemo-helm-readme.md.gotmpl  | 11 +++++++++++
 k8s/helm/templates/_helpers.tpl                    |  4 ++++
 .../templates/clickhouse/clickhouse-secret.yaml    | 14 +++++++++++---
 k8s/helm/values.yaml                               |  4 ++--
 tools/lint/lint-helm.sh                            | 13 +++++++++++++
 6 files changed, 53 insertions(+), 6 deletions(-)

diff --git a/k8s/helm/README.md b/k8s/helm/README.md
index ce5b3cfb01..7ce559e2fb 100644
--- a/k8s/helm/README.md
+++ b/k8s/helm/README.md
@@ -40,6 +40,17 @@ non-critical single-node installations. It does not provide replication or
 automatic backups. For a production deployment that requires high availability,
 set `clickhouse.enabled` to `false` and provide a separately managed ClickHouse:
 
+First, create the credentials Secret in the Helm release namespace. The Secret
+key must match `externalClickhouse.existingSecretPasswordKey`:
+
+```shell
+kubectl create secret generic clickhouse-credentials \
+  --namespace <release-namespace> \
+  --from-literal=password='<clickhouse-password>'
+```
+
+Then configure the external connection:
+
 ```yaml
 clickhouse:
   enabled: false
@@ -179,7 +190,7 @@ and
 | clickhouse.auth.database | string | `"intake"` | ClickHouse database used by Intake. |
 | clickhouse.auth.existingSecret | string | `""` | Name of an existing Secret containing the ClickHouse password. If empty, the chart creates one. |
 | clickhouse.auth.existingSecretPasswordKey | string | `"password"` | Key in auth.existingSecret containing the ClickHouse password. |
-| clickhouse.auth.password | string | `"nemo"` | ClickHouse password used when auth.existingSecret is empty. |
+| clickhouse.auth.password | string | `""` | ClickHouse password used when auth.existingSecret is empty. If empty, the chart generates a random password. |
 | clickhouse.auth.username | string | `"nemo"` | ClickHouse username used by Intake. |
 | clickhouse.enabled | bool | `true` | Whether to deploy the embedded ClickHouse. Set to false to use `externalClickhouse`. |
 | clickhouse.image.pullPolicy | string | `"IfNotPresent"` | ClickHouse image pull policy. |
diff --git a/k8s/helm/helm-docs-template/nemo-helm-readme.md.gotmpl b/k8s/helm/helm-docs-template/nemo-helm-readme.md.gotmpl
index 93c0061ade..7b04f6ee97 100644
--- a/k8s/helm/helm-docs-template/nemo-helm-readme.md.gotmpl
+++ b/k8s/helm/helm-docs-template/nemo-helm-readme.md.gotmpl
@@ -40,6 +40,17 @@ non-critical single-node installations. It does not provide replication or
 automatic backups. For a production deployment that requires high availability,
 set `clickhouse.enabled` to `false` and provide a separately managed ClickHouse:
 
+First, create the credentials Secret in the Helm release namespace. The Secret
+key must match `externalClickhouse.existingSecretPasswordKey`:
+
+```shell
+kubectl create secret generic clickhouse-credentials \
+  --namespace <release-namespace> \
+  --from-literal=password='<clickhouse-password>'
+```
+
+Then configure the external connection:
+
 ```yaml
 clickhouse:
   enabled: false
diff --git a/k8s/helm/templates/_helpers.tpl b/k8s/helm/templates/_helpers.tpl
index 60ecd15110..ec16b0108e 100644
--- a/k8s/helm/templates/_helpers.tpl
+++ b/k8s/helm/templates/_helpers.tpl
@@ -420,8 +420,12 @@ Key containing the ClickHouse password used by Intake.
 */}}
 {{- define "nemo-common.clickhouse.passwordKey" -}}
 {{- if .Values.clickhouse.enabled -}}
+{{- if .Values.clickhouse.auth.existingSecret -}}
 {{- .Values.clickhouse.auth.existingSecretPasswordKey | default "password" -}}
 {{- else -}}
+password
+{{- end -}}
+{{- else -}}
 {{- required "externalClickhouse.existingSecretPasswordKey is required when clickhouse.enabled=false" .Values.externalClickhouse.existingSecretPasswordKey -}}
 {{- end -}}
 {{- end -}}
diff --git a/k8s/helm/templates/clickhouse/clickhouse-secret.yaml b/k8s/helm/templates/clickhouse/clickhouse-secret.yaml
index 7bc42fc9e8..69e71fea2f 100644
--- a/k8s/helm/templates/clickhouse/clickhouse-secret.yaml
+++ b/k8s/helm/templates/clickhouse/clickhouse-secret.yaml
@@ -1,12 +1,20 @@
 {{- if and .Values.clickhouse.enabled (not .Values.clickhouse.auth.existingSecret) }}
+{{- $secretName := include "nemo-common.clickhouse.fullname" . }}
+{{- $passwordData := randAlphaNum 32 | b64enc }}
+{{- with (lookup "v1" "Secret" .Release.Namespace $secretName) }}
+{{- $passwordData = index .data "password" | default $passwordData }}
+{{- end }}
+{{- with .Values.clickhouse.auth.password }}
+{{- $passwordData = . | b64enc }}
+{{- end }}
 apiVersion: v1
 kind: Secret
 metadata:
-  name: {{ include "nemo-common.clickhouse.fullname" . }}
+  name: {{ $secretName }}
   labels:
     app.kubernetes.io/component: clickhouse
     {{- include "nemo-platform.labels" . | nindent 4 }}
 type: Opaque
-stringData:
-  password: {{ .Values.clickhouse.auth.password | quote }}
+data:
+  password: {{ $passwordData | quote }}
 {{- end }}
diff --git a/k8s/helm/values.yaml b/k8s/helm/values.yaml
index e58b2f7615..6816708238 100644
--- a/k8s/helm/values.yaml
+++ b/k8s/helm/values.yaml
@@ -260,8 +260,8 @@ clickhouse:
   auth:
     # -- ClickHouse username used by Intake.
     username: nemo
-    # -- ClickHouse password used when auth.existingSecret is empty.
-    password: nemo
+    # -- ClickHouse password used when auth.existingSecret is empty. If empty, the chart generates a random password.
+    password: ""
     # -- ClickHouse database used by Intake.
     database: intake
     # -- Name of an existing Secret containing the ClickHouse password. If empty, the chart creates one.
diff --git a/tools/lint/lint-helm.sh b/tools/lint/lint-helm.sh
index cb6c3b7520..8d8d2e1cfc 100755
--- a/tools/lint/lint-helm.sh
+++ b/tools/lint/lint-helm.sh
@@ -54,6 +54,19 @@ external_clickhouse_secret_output=$(helm template "${HELM_RELEASE_NAME}" "${HELM
 grep -Fq "externalClickhouse.existingSecret is required when clickhouse.enabled=false" \
   <<<"${external_clickhouse_secret_output}"
 
+# Generated embedded credentials must never use the shipped username as a known
+# password, and all consumers must reference the generated `password` key.
+generated_clickhouse_output=$(helm template "${HELM_RELEASE_NAME}" "${HELM_FOLDER}" \
+  --set clickhouse.auth.existingSecretPasswordKey=not-the-generated-key)
+if grep -Fq "bmVtbw==" <<<"${generated_clickhouse_output}"; then
+  echo "Embedded ClickHouse rendered the known 'nemo' password" >&2
+  exit 1
+fi
+if grep -Fq "not-the-generated-key" <<<"${generated_clickhouse_output}"; then
+  echo "Embedded ClickHouse referenced a key not created by its generated Secret" >&2
+  exit 1
+fi
+
 # Validate the Helm chart by rendering templates with all values files in ci/ directory
 shopt -s nullglob
 for value_file in "${HELM_FOLDER}"/ci/*.yaml; do

From fc70535847d8c40c47bec59318cb6bf2c27a6ba4 Mon Sep 17 00:00:00 2001
From: Brian Newsom <brnewsom@nvidia.com>
Date: Thu, 30 Jul 2026 13:15:46 -0600
Subject: [PATCH 3/6] test(helm): validate external clickhouse password key

Signed-off-by: Brian Newsom <brnewsom@nvidia.com>
---
 tools/lint/lint-helm.sh | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/tools/lint/lint-helm.sh b/tools/lint/lint-helm.sh
index 8d8d2e1cfc..3eab530aed 100755
--- a/tools/lint/lint-helm.sh
+++ b/tools/lint/lint-helm.sh
@@ -54,6 +54,16 @@ external_clickhouse_secret_output=$(helm template "${HELM_RELEASE_NAME}" "${HELM
 grep -Fq "externalClickhouse.existingSecret is required when clickhouse.enabled=false" \
   <<<"${external_clickhouse_secret_output}"
 
+external_clickhouse_password_key_output=$(helm template "${HELM_RELEASE_NAME}" "${HELM_FOLDER}" \
+  --set clickhouse.enabled=false \
+  --set externalClickhouse.host=clickhouse.example.internal \
+  --set externalClickhouse.existingSecret=clickhouse-credentials 2>&1) && {
+  echo "External ClickHouse accepted a missing password key" >&2
+  exit 1
+}
+grep -Fq "externalClickhouse.existingSecretPasswordKey is required when clickhouse.enabled=false" \
+  <<<"${external_clickhouse_password_key_output}"
+
 # Generated embedded credentials must never use the shipped username as a known
 # password, and all consumers must reference the generated `password` key.
 generated_clickhouse_output=$(helm template "${HELM_RELEASE_NAME}" "${HELM_FOLDER}" \

From 1f69ea9874460541dcb3f685b77c7d3f64f0e743 Mon Sep 17 00:00:00 2001
From: Brian Newsom <brnewsom@nvidia.com>
Date: Thu, 30 Jul 2026 13:21:28 -0600
Subject: [PATCH 4/6] fix(helm): leave clickhouse password unset

Signed-off-by: Brian Newsom <brnewsom@nvidia.com>
---
 k8s/helm/README.md   | 2 +-
 k8s/helm/values.yaml | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/k8s/helm/README.md b/k8s/helm/README.md
index 7ce559e2fb..a900b16e38 100644
--- a/k8s/helm/README.md
+++ b/k8s/helm/README.md
@@ -190,7 +190,7 @@ and
 | clickhouse.auth.database | string | `"intake"` | ClickHouse database used by Intake. |
 | clickhouse.auth.existingSecret | string | `""` | Name of an existing Secret containing the ClickHouse password. If empty, the chart creates one. |
 | clickhouse.auth.existingSecretPasswordKey | string | `"password"` | Key in auth.existingSecret containing the ClickHouse password. |
-| clickhouse.auth.password | string | `""` | ClickHouse password used when auth.existingSecret is empty. If empty, the chart generates a random password. |
+| clickhouse.auth.password | string | `nil` | ClickHouse password used when auth.existingSecret is empty. If unset, the chart generates a random password. |
 | clickhouse.auth.username | string | `"nemo"` | ClickHouse username used by Intake. |
 | clickhouse.enabled | bool | `true` | Whether to deploy the embedded ClickHouse. Set to false to use `externalClickhouse`. |
 | clickhouse.image.pullPolicy | string | `"IfNotPresent"` | ClickHouse image pull policy. |
diff --git a/k8s/helm/values.yaml b/k8s/helm/values.yaml
index 6816708238..922f8274de 100644
--- a/k8s/helm/values.yaml
+++ b/k8s/helm/values.yaml
@@ -260,8 +260,8 @@ clickhouse:
   auth:
     # -- ClickHouse username used by Intake.
     username: nemo
-    # -- ClickHouse password used when auth.existingSecret is empty. If empty, the chart generates a random password.
-    password: ""
+    # -- ClickHouse password used when auth.existingSecret is empty. If unset, the chart generates a random password.
+    password: null
     # -- ClickHouse database used by Intake.
     database: intake
     # -- Name of an existing Secret containing the ClickHouse password. If empty, the chart creates one.

From 556dfed29ff346ec5e8c3f199fa7dbe9bdc371aa Mon Sep 17 00:00:00 2001
From: Brian Newsom <brnewsom@nvidia.com>
Date: Thu, 30 Jul 2026 15:28:59 -0600
Subject: [PATCH 5/6] fix(helm): make clickhouse statefulset upgrade-safe

Signed-off-by: Brian Newsom <brnewsom@nvidia.com>
---
 k8s/helm/README.md                            |  3 +++
 .../clickhouse/clickhouse-statefulset.yaml    | 23 +++-------------
 k8s/helm/values.yaml                          | 27 +++++++++++++++++++
 tools/lint/lint-helm.sh                       | 15 +++++++++++
 4 files changed, 49 insertions(+), 19 deletions(-)

diff --git a/k8s/helm/README.md b/k8s/helm/README.md
index a900b16e38..7f1ba1caa2 100644
--- a/k8s/helm/README.md
+++ b/k8s/helm/README.md
@@ -196,6 +196,7 @@ and
 | clickhouse.image.pullPolicy | string | `"IfNotPresent"` | ClickHouse image pull policy. |
 | clickhouse.image.repository | string | `"docker.io/clickhouse/clickhouse-server"` | ClickHouse image repository. Intake is tested against the 26.3 LTS release line. |
 | clickhouse.image.tag | string | `"26.3"` | ClickHouse image tag. |
+| clickhouse.livenessProbe | object | This object has the following default values for the liveness probe configuration. | Liveness probe configuration for the ClickHouse container. |
 | clickhouse.nodeSelector | object | `{}` | Node selector for the ClickHouse pod. |
 | clickhouse.persistence.enabled | bool | `true` | Whether to persist embedded ClickHouse data. |
 | clickhouse.persistence.size | string | `"20Gi"` | PersistentVolumeClaim size. See the Intake and ClickHouse sizing guidance in the chart README. |
@@ -203,6 +204,7 @@ and
 | clickhouse.podAnnotations | object | `{}` | Annotations to add to the ClickHouse pod. |
 | clickhouse.podLabels | object | `{}` | Additional labels to add to the ClickHouse pod. |
 | clickhouse.podSecurityContext | object | `{}` | Optional pod security context for the ClickHouse pod. |
+| clickhouse.readinessProbe | object | This object has the following default values for the readiness probe configuration. | Readiness probe configuration for the ClickHouse container. |
 | clickhouse.resources | object | `{"requests":{"cpu":"2","memory":"8Gi"}}` | Resource requests and limits for the ClickHouse container. The defaults are the supported small-volume starting point. |
 | clickhouse.securityContext | object | `{}` | Optional container security context for the ClickHouse container. |
 | clickhouse.service.annotations | object | `{}` | Annotations to add to the ClickHouse Service. |
@@ -213,6 +215,7 @@ and
 | clickhouse.serviceAccount.automount | bool | `false` | Automatically mount the ServiceAccount's API credentials. |
 | clickhouse.serviceAccount.create | bool | `true` | Specifies whether a service account should be created for the ClickHouse pod. |
 | clickhouse.serviceAccount.name | string | `""` | The name of the service account to use. If not set and create is true, a name is generated from the release fullname. |
+| clickhouse.startupProbe | object | This object has the following default values for the startup probe configuration. | Startup probe configuration for the ClickHouse container. |
 | clickhouse.tolerations | list | `[]` | Tolerations for the ClickHouse pod. |
 | core | object | This object has the following default values for the core deployment configuration. | Core deployment configuration settings |
 | core.controller.affinity | object | `{}` | Affinity configuration for the controller service. |
diff --git a/k8s/helm/templates/clickhouse/clickhouse-statefulset.yaml b/k8s/helm/templates/clickhouse/clickhouse-statefulset.yaml
index 7011c1c058..409f2a107a 100644
--- a/k8s/helm/templates/clickhouse/clickhouse-statefulset.yaml
+++ b/k8s/helm/templates/clickhouse/clickhouse-statefulset.yaml
@@ -76,26 +76,11 @@ spec:
               containerPort: 9000
               protocol: TCP
           startupProbe:
-            httpGet:
-              path: /ping
-              port: http
-            periodSeconds: 5
-            timeoutSeconds: 3
-            failureThreshold: 60
+            {{- toYaml .Values.clickhouse.startupProbe | nindent 12 }}
           livenessProbe:
-            httpGet:
-              path: /ping
-              port: http
-            periodSeconds: 10
-            timeoutSeconds: 5
-            failureThreshold: 3
+            {{- toYaml .Values.clickhouse.livenessProbe | nindent 12 }}
           readinessProbe:
-            httpGet:
-              path: /ping
-              port: http
-            periodSeconds: 5
-            timeoutSeconds: 3
-            failureThreshold: 3
+            {{- toYaml .Values.clickhouse.readinessProbe | nindent 12 }}
           volumeMounts:
             - name: data
               mountPath: /var/lib/clickhouse
@@ -112,7 +97,7 @@ spec:
         name: data
         labels:
           app.kubernetes.io/component: clickhouse
-          {{- include "nemo-platform.labels" . | nindent 10 }}
+          {{- include "nemo-platform.selectorLabels" . | nindent 10 }}
       spec:
         accessModes:
           - ReadWriteOnce
diff --git a/k8s/helm/values.yaml b/k8s/helm/values.yaml
index 922f8274de..3f3a6cd495 100644
--- a/k8s/helm/values.yaml
+++ b/k8s/helm/values.yaml
@@ -287,6 +287,33 @@ clickhouse:
     requests:
       cpu: "2"
       memory: 8Gi
+  # -- Startup probe configuration for the ClickHouse container.
+  # @default -- This object has the following default values for the startup probe configuration.
+  startupProbe:
+    httpGet:
+      path: /ping
+      port: http
+    periodSeconds: 5
+    timeoutSeconds: 3
+    failureThreshold: 60
+  # -- Liveness probe configuration for the ClickHouse container.
+  # @default -- This object has the following default values for the liveness probe configuration.
+  livenessProbe:
+    httpGet:
+      path: /ping
+      port: http
+    periodSeconds: 10
+    timeoutSeconds: 5
+    failureThreshold: 3
+  # -- Readiness probe configuration for the ClickHouse container.
+  # @default -- This object has the following default values for the readiness probe configuration.
+  readinessProbe:
+    httpGet:
+      path: /ping
+      port: http
+    periodSeconds: 5
+    timeoutSeconds: 3
+    failureThreshold: 3
   # -- Optional pod security context for the ClickHouse pod.
   podSecurityContext: {}
   # -- Optional container security context for the ClickHouse container.
diff --git a/tools/lint/lint-helm.sh b/tools/lint/lint-helm.sh
index 3eab530aed..59b6d4c243 100755
--- a/tools/lint/lint-helm.sh
+++ b/tools/lint/lint-helm.sh
@@ -77,6 +77,21 @@ if grep -Fq "not-the-generated-key" <<<"${generated_clickhouse_output}"; then
   exit 1
 fi
 
+# ClickHouse probe overrides must render, and PVC template labels must remain
+# stable across chart and application version upgrades.
+clickhouse_statefulset_output=$(helm template "${HELM_RELEASE_NAME}" "${HELM_FOLDER}" \
+  --show-only templates/clickhouse/clickhouse-statefulset.yaml \
+  --set clickhouse.startupProbe.periodSeconds=17)
+grep -A6 -F "startupProbe:" <<<"${clickhouse_statefulset_output}" \
+  | grep -Fq "periodSeconds: 17"
+clickhouse_pvc_template=$(sed -n '/^  volumeClaimTemplates:/,$p' <<<"${clickhouse_statefulset_output}")
+if grep -Eq "helm.sh/chart|app.kubernetes.io/version" <<<"${clickhouse_pvc_template}"; then
+  echo "ClickHouse PVC template contains labels that change across chart upgrades" >&2
+  exit 1
+fi
+grep -Fq "app.kubernetes.io/component: clickhouse" <<<"${clickhouse_pvc_template}"
+grep -Fq "app.kubernetes.io/instance: ${HELM_RELEASE_NAME}" <<<"${clickhouse_pvc_template}"
+
 # Validate the Helm chart by rendering templates with all values files in ci/ directory
 shopt -s nullglob
 for value_file in "${HELM_FOLDER}"/ci/*.yaml; do

From 7b141db5c47043278bbc0f7faa2fff2431db6330 Mon Sep 17 00:00:00 2001
From: Brian Newsom <brnewsom@nvidia.com>
Date: Thu, 30 Jul 2026 15:36:19 -0600
Subject: [PATCH 6/6] fix(helm): roll workloads on clickhouse credential
 changes

Signed-off-by: Brian Newsom <brnewsom@nvidia.com>
---
 k8s/helm/templates/_helpers.tpl               | 22 +++++++++++++++++++
 k8s/helm/templates/api/api-deployment.yaml    |  1 +
 .../clickhouse/clickhouse-statefulset.yaml    |  5 +++--
 tools/lint/lint-helm.sh                       | 22 +++++++++++++++++++
 4 files changed, 48 insertions(+), 2 deletions(-)

diff --git a/k8s/helm/templates/_helpers.tpl b/k8s/helm/templates/_helpers.tpl
index ec16b0108e..54d8cf256b 100644
--- a/k8s/helm/templates/_helpers.tpl
+++ b/k8s/helm/templates/_helpers.tpl
@@ -430,6 +430,28 @@ password
 {{- end -}}
 {{- end -}}
 
+{{/*
+Checksum of the ClickHouse password used by Intake. Prefer an explicitly
+configured password for a chart-managed Secret, then the live Secret value, so
+credential changes roll both API and ClickHouse pods together. During first
+install with a generated password, use a deterministic fallback until the
+Secret exists.
+*/}}
+{{- define "nemo-common.clickhouse.credentialsChecksum" -}}
+{{- $_ := include "nemo-common.clickhouse.url" . -}}
+{{- $secretName := include "nemo-common.clickhouse.secretName" . -}}
+{{- $passwordKey := include "nemo-common.clickhouse.passwordKey" . -}}
+{{- $secret := lookup "v1" "Secret" .Release.Namespace $secretName -}}
+{{- $passwordData := get ($secret.data | default dict) $passwordKey | default "" -}}
+{{- if and .Values.clickhouse.enabled (not .Values.clickhouse.auth.existingSecret) .Values.clickhouse.auth.password -}}
+{{- .Values.clickhouse.auth.password | toString | b64enc | sha256sum -}}
+{{- else if $passwordData -}}
+{{- $passwordData | sha256sum -}}
+{{- else -}}
+{{- printf "%s:%s" $secretName $passwordKey | sha256sum -}}
+{{- end -}}
+{{- end -}}
+
 {{/*
 nemo-common.otel-env generates an env var array from the top-level telemetry configuration.
 Follows the specification at https://opentelemetry.io/docs/specs/otel/configuration/sdk-environment-variables/
diff --git a/k8s/helm/templates/api/api-deployment.yaml b/k8s/helm/templates/api/api-deployment.yaml
index f24659de37..cfd2ca4a20 100644
--- a/k8s/helm/templates/api/api-deployment.yaml
+++ b/k8s/helm/templates/api/api-deployment.yaml
@@ -27,6 +27,7 @@ spec:
         {{- toYaml . | nindent 8 }}
         {{- end }}
         {{- include "nemo-platform.podAnnotations" . | nindent 8 }}
+        checksum/clickhouse-credentials: {{ include "nemo-common.clickhouse.credentialsChecksum" . }}
       labels:
         app.kubernetes.io/component: nmp-api
         {{- include "nemo-platform.labels" . | nindent 8 }}
diff --git a/k8s/helm/templates/clickhouse/clickhouse-statefulset.yaml b/k8s/helm/templates/clickhouse/clickhouse-statefulset.yaml
index 409f2a107a..16df992b78 100644
--- a/k8s/helm/templates/clickhouse/clickhouse-statefulset.yaml
+++ b/k8s/helm/templates/clickhouse/clickhouse-statefulset.yaml
@@ -20,10 +20,11 @@ spec:
       {{- include "nemo-platform.selectorLabels" . | nindent 6 }}
   template:
     metadata:
-      {{- with .Values.clickhouse.podAnnotations }}
       annotations:
+        checksum/clickhouse-credentials: {{ include "nemo-common.clickhouse.credentialsChecksum" . }}
+        {{- with .Values.clickhouse.podAnnotations }}
         {{- toYaml . | nindent 8 }}
-      {{- end }}
+        {{- end }}
       labels:
         app.kubernetes.io/component: clickhouse
         {{- include "nemo-platform.labels" . | nindent 8 }}
diff --git a/tools/lint/lint-helm.sh b/tools/lint/lint-helm.sh
index 59b6d4c243..882b069aed 100755
--- a/tools/lint/lint-helm.sh
+++ b/tools/lint/lint-helm.sh
@@ -82,8 +82,30 @@ fi
 clickhouse_statefulset_output=$(helm template "${HELM_RELEASE_NAME}" "${HELM_FOLDER}" \
   --show-only templates/clickhouse/clickhouse-statefulset.yaml \
   --set clickhouse.startupProbe.periodSeconds=17)
+api_deployment_output=$(helm template "${HELM_RELEASE_NAME}" "${HELM_FOLDER}" \
+  --show-only templates/api/api-deployment.yaml)
+rotated_api_deployment_output=$(helm template "${HELM_RELEASE_NAME}" "${HELM_FOLDER}" \
+  --show-only templates/api/api-deployment.yaml \
+  --set-string clickhouse.auth.password=rotated-test-password)
 grep -A6 -F "startupProbe:" <<<"${clickhouse_statefulset_output}" \
   | grep -Fq "periodSeconds: 17"
+clickhouse_credentials_checksum=$(awk \
+  '$1 == "checksum/clickhouse-credentials:" { print $2; exit }' \
+  <<<"${clickhouse_statefulset_output}")
+api_credentials_checksum=$(awk \
+  '$1 == "checksum/clickhouse-credentials:" { print $2; exit }' \
+  <<<"${api_deployment_output}")
+rotated_credentials_checksum=$(awk \
+  '$1 == "checksum/clickhouse-credentials:" { print $2; exit }' \
+  <<<"${rotated_api_deployment_output}")
+if [[ -z "${clickhouse_credentials_checksum}" || "${clickhouse_credentials_checksum}" != "${api_credentials_checksum}" ]]; then
+  echo "API and ClickHouse workloads do not share the same credential checksum" >&2
+  exit 1
+fi
+if [[ -z "${rotated_credentials_checksum}" || "${rotated_credentials_checksum}" == "${api_credentials_checksum}" ]]; then
+  echo "ClickHouse credential checksum did not change with the configured password" >&2
+  exit 1
+fi
 clickhouse_pvc_template=$(sed -n '/^  volumeClaimTemplates:/,$p' <<<"${clickhouse_statefulset_output}")
 if grep -Eq "helm.sh/chart|app.kubernetes.io/version" <<<"${clickhouse_pvc_template}"; then
   echo "ClickHouse PVC template contains labels that change across chart upgrades" >&2