From 67b8f0bd80c4aa4c36d538b9854b264db2c30ebd Mon Sep 17 00:00:00 2001 From: Kristina Pathak Date: Tue, 28 Jul 2026 13:13:58 -0700 Subject: [PATCH 01/13] feat(self-managed): add profile-based observability Signed-off-by: Kristina Pathak --- .../dev/skills/nvcf-explore-stack/SKILL.md | 16 +- deploy/helm/function-autoscaler/Chart.yaml | 2 +- .../templates/deployment.yaml | 7 +- deploy/helm/function-autoscaler/values.yaml | 7 + deploy/stacks/observability/AGENTS.md | 38 +-- deploy/stacks/observability/Makefile | 13 +- deploy/stacks/observability/README.md | 246 ++++++++++++------ .../charts/nvcf-default-monitors/Chart.yaml | 4 +- .../templates/dcgm-podmonitor.yaml | 2 +- .../templates/nvca-servicemonitor.yaml | 39 +++ .../templates/worker-podmonitor.yaml | 38 +++ .../charts/nvcf-default-monitors/values.yaml | 44 +++- .../nvcf-observability-contract/Chart.yaml | 21 ++ .../templates/autoscaler-configmap.yaml | 42 +++ .../templates/profile-configmap.yaml | 36 +++ .../nvcf-observability-contract/values.yaml | 32 +++ .../charts/nvcf-otel-collector/Chart.yaml | 6 +- .../charts/nvcf-otel-collector/values.yaml | 2 +- .../observability/environments/base.yaml | 94 +++++-- .../observability/environments/local.yaml | 20 +- .../helmfile.d/01-observability.yaml.gotmpl | 167 ++++++++++-- .../observability/tests/profile-defaults.sh | 229 ++++++++++++++++ .../self-managed/environments/base.yaml | 39 ++- deploy/stacks/self-managed/global.yaml.gotmpl | 38 +++ ...0-observability-infrastructure.yaml.gotmpl | 10 +- .../helmfile.d/03-observability.yaml.gotmpl | 20 ++ .../tests/observability-autoscaler.sh | 116 +++++++++ 27 files changed, 1109 insertions(+), 219 deletions(-) create mode 100644 deploy/stacks/observability/charts/nvcf-default-monitors/templates/nvca-servicemonitor.yaml create mode 100644 deploy/stacks/observability/charts/nvcf-default-monitors/templates/worker-podmonitor.yaml create mode 100644 deploy/stacks/observability/charts/nvcf-observability-contract/Chart.yaml create mode 100644 deploy/stacks/observability/charts/nvcf-observability-contract/templates/autoscaler-configmap.yaml create mode 100644 deploy/stacks/observability/charts/nvcf-observability-contract/templates/profile-configmap.yaml create mode 100644 deploy/stacks/observability/charts/nvcf-observability-contract/values.yaml create mode 100755 deploy/stacks/observability/tests/profile-defaults.sh create mode 100755 deploy/stacks/self-managed/tests/observability-autoscaler.sh diff --git a/ai-tooling/dev/skills/nvcf-explore-stack/SKILL.md b/ai-tooling/dev/skills/nvcf-explore-stack/SKILL.md index 6438e924a..4e5fa9668 100644 --- a/ai-tooling/dev/skills/nvcf-explore-stack/SKILL.md +++ b/ai-tooling/dev/skills/nvcf-explore-stack/SKILL.md @@ -14,13 +14,13 @@ description: >- license: Apache-2.0 compatibility: Requires a local checkout of the NVCF monorepo with deploy/stacks/self-managed/ and deploy/stacks/nvcf-compute-plane/ present author: "nvcf-core-eng " -version: "1.0.0" +version: "1.1.0" tags: [nvcf, self-managed, self-hosted, helmfile, deployment, stack-topology] tools: [Read, Grep, Glob] metadata: internal: false author: "nvcf-core-eng " - version: "1.0.0" + version: "1.1.0" tags: [nvcf, self-managed, self-hosted, helmfile, deployment, stack-topology, dependencies, hooks] languages: [yaml, markdown] frameworks: [helmfile, helm, kubernetes] @@ -39,17 +39,21 @@ Use this skill long enough to answer the question, then hand off to the right ex ## Required inputs -Read these from the monorepo root (the directory containing `imports.yaml`): +Read these from the monorepo root. Internal checkouts include `imports.yaml`; +the public snapshot may omit it. Authoritative (always read first when answering): +- `deploy/stacks/self-managed/helmfile.d/00-observability-infrastructure.yaml.gotmpl` - `deploy/stacks/self-managed/helmfile.d/01-dependencies.yaml.gotmpl` - `deploy/stacks/self-managed/helmfile.d/02-core.yaml.gotmpl` - `deploy/stacks/self-managed/helmfile.d/03-observability.yaml.gotmpl` +- `deploy/stacks/observability/helmfile.d/01-observability.yaml.gotmpl` - `deploy/stacks/nvcf-compute-plane/helmfile.d/01-dependencies.yaml.gotmpl` - `deploy/stacks/nvcf-compute-plane/helmfile.d/02-nvca.yaml.gotmpl` -Provenance (when asked which subtree is monorepo-native vs. upstream-owned): +Provenance (when asked which subtree is monorepo-native vs. upstream-owned and +the file is present): - `imports.yaml` @@ -68,13 +72,13 @@ What deploys X : Look up release `X` in the helmfile stage files. Return chart name, version, namespace, and which gotmpl file declares it. If the chart is checked in, also point at `deploy/helm//`. What does X depend on -: Return the `needs:` chain for that release plus the stage gate it sits behind (control-plane stages 1 -> 2 -> 3, then compute-plane). Include any `condition:` that gates whether X deploys at all. +: Return the `needs:` chain for that release plus the stage gate it sits behind (control-plane stages 0 -> 1 -> 2 -> 3, then compute-plane). Include any profile, `condition:`, or component mode that gates whether X deploys at all. What hooks run for X : Read the checked-in chart under `deploy/helm//` when available. Search its `templates/` directory for Helm hook annotations, weights, hook events (pre-install / post-install), images used, and purpose. Cite the chart-relative template file path. If the chart is consumed from OCI only and the local workspace has routing metadata, use the workspace metadata and cite it. Walk me through the full deployment order -: Summarize control-plane stages 0 through 3 from the self-managed gotmpl file headers. Then summarize the compute-plane stage from `deploy/stacks/nvcf-compute-plane/helmfile.d/01-dependencies.yaml.gotmpl` and `deploy/stacks/nvcf-compute-plane/helmfile.d/02-nvca.yaml.gotmpl`. Call out which releases run in parallel inside a stage and which are serialized by `needs:`. +: Summarize control-plane stages 0 through 3 from the self-managed gotmpl files. Stage 0 delegates to the shared observability Helmfile when `observability.profile` is enabled. Stage 3 installs State Metrics and the function autoscaler for `control` and `all`. Then summarize the compute-plane stage from `deploy/stacks/nvcf-compute-plane/helmfile.d/01-dependencies.yaml.gotmpl` and `deploy/stacks/nvcf-compute-plane/helmfile.d/02-nvca.yaml.gotmpl`. Call out which releases run in parallel inside a stage and which are serialized by `needs:`. Which subtree do I edit to change X : Two answers, both are important. For chart wiring (Helm hooks, manifests, values, hook weights) point at `deploy/helm//` if the chart is checked in, or note `oci-only` (the chart is consumed from the OCI registry and does not live in the monorepo). For runtime application logic, use the chart image repository, imports.yaml, and any available workspace routing metadata. `authoritative_source: native` means edits land here. `upstream` means edits also flow back to the upstream repo through the internal source-sync workflow. diff --git a/deploy/helm/function-autoscaler/Chart.yaml b/deploy/helm/function-autoscaler/Chart.yaml index 0c824eb80..482ff68fe 100644 --- a/deploy/helm/function-autoscaler/Chart.yaml +++ b/deploy/helm/function-autoscaler/Chart.yaml @@ -19,4 +19,4 @@ description: A Helm chart for NVCF Function Autoscaler Service type: application version: 0.0.0 # autoversioning enabled via release pipeline -appVersion: "0.0.0" +appVersion: "1.18.3" diff --git a/deploy/helm/function-autoscaler/templates/deployment.yaml b/deploy/helm/function-autoscaler/templates/deployment.yaml index 5f55f50de..2b0fb675e 100644 --- a/deploy/helm/function-autoscaler/templates/deployment.yaml +++ b/deploy/helm/function-autoscaler/templates/deployment.yaml @@ -66,11 +66,16 @@ spec: {{- toYaml .Values.functionautoscaler.readinessProbe | nindent 10 }} resources: {{- toYaml .Values.functionautoscaler.resources | nindent 10 }} - {{- if .Values.functionautoscaler.env }} + {{- if or .Values.functionautoscaler.env .Values.functionautoscaler.envFrom }} envFrom: + {{- if .Values.functionautoscaler.env }} - configMapRef: name: {{ include "function-autoscaler.fullname" . }}-env {{- end }} + {{- with .Values.functionautoscaler.envFrom }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} {{- if or .Values.functionautoscaler.volumeMounts .Values.functionautoscaler.vault.enabled }} volumeMounts: {{- with .Values.functionautoscaler.volumeMounts }} diff --git a/deploy/helm/function-autoscaler/values.yaml b/deploy/helm/function-autoscaler/values.yaml index db8eec8f7..dd5e82767 100644 --- a/deploy/helm/function-autoscaler/values.yaml +++ b/deploy/helm/function-autoscaler/values.yaml @@ -140,3 +140,10 @@ functionautoscaler: # Read the secrets file rendered by the Vault Agent. Override this value # when mounting secrets manually. SECRETS_PATH: "/vault/secrets/secrets.json" + + # Additional ConfigMaps or Secrets to load after the chart-owned env + # ConfigMap. This lets a stack provide resolved infrastructure contracts + # without duplicating them in this chart's values. + envFrom: [] + # - configMapRef: + # name: nvcf-observability-autoscaler diff --git a/deploy/stacks/observability/AGENTS.md b/deploy/stacks/observability/AGENTS.md index b8788476f..e88dafa56 100644 --- a/deploy/stacks/observability/AGENTS.md +++ b/deploy/stacks/observability/AGENTS.md @@ -2,9 +2,9 @@ ## Purpose -This is the shared NVCF observability Helmfile stack scaffold. It owns optional -cluster-level observability infrastructure intended for self-managed -control-plane and compute-plane deployments once consuming stack wiring lands. +This is the shared NVCF observability Helmfile stack. It owns cluster-level +observability infrastructure for self-managed control-plane and compute-plane +deployments. ## Scope @@ -14,13 +14,13 @@ control-plane and compute-plane deployments once consuming stack wiring lands. - Keep application metrics endpoints, labels, port names, and paths in the application charts that expose those endpoints. - Do not add control-plane services or compute-plane services here. -- Keep base runtime gates off until a consuming stack owns chart distribution, - mirrored images, and environment overlays. -- Consumer overlays for self-managed control-plane deployments may enable the - control-plane collector, OTel Operator, Prometheus Operator CRDs, and bundled - TSDB contract together. -- Keep BYOO and NVCA collectors default off unless a consumer explicitly - enables them through values. +- Keep the reusable base profile `disabled`. +- Derive control-plane and compute-plane behavior from + `observability.profile`; do not expose parallel plane booleans. +- Enabled profiles install shared components once by default. Fine-grained + component modes are `install`, `existing`, and `disabled`. +- Derive NVCA BYOO support from compute and all profiles. Keep the NVCA + collector disabled by default. ## Key Files @@ -28,16 +28,16 @@ control-plane and compute-plane deployments once consuming stack wiring lands. - `environments/base.yaml`: scaffold defaults and scrape contracts - `charts/nvcf-otel-collector`: OpenTelemetryCollector resource and Target Allocator RBAC - `charts/nvcf-default-monitors`: centrally owned concrete monitor resources +- `charts/nvcf-observability-contract`: resolved settings for consumers such as the autoscaler - `values/victoria-metrics.yaml.gotmpl`: values bridge for the VictoriaMetrics chart -## Modes +## Profiles -- `install`: this stack installs and owns enabled shared infrastructure. -- `existing`: this stack may render default monitors, but shared infrastructure - is owned by another stack instance. -- `disabled`: this stack renders no observability runtime resources. This is - not a supported self-managed control-plane autoscaler mode, but is valid for - compute-plane deployments that do not need NVCF-managed scrape coverage. +- `disabled`: render no observability runtime resources. +- `control`: install control-plane metrics and autoscaler backend defaults. +- `compute`: install NVCA, DCGM, and worker metric defaults. +- `all`: install the union once. -Do not use Helm `lookup` to silently change modes based on live cluster state. -The consuming stack must choose the mode explicitly. +Do not use Helm `lookup` to silently change profiles or component ownership +based on live cluster state. The consuming stack must choose the profile +explicitly. diff --git a/deploy/stacks/observability/Makefile b/deploy/stacks/observability/Makefile index 2e6b2075c..706c25ec5 100644 --- a/deploy/stacks/observability/Makefile +++ b/deploy/stacks/observability/Makefile @@ -9,7 +9,7 @@ KUBECONFIG_FLAG = $(if $(KUBECONFIG_FILE),--kubeconfig $(KUBECONFIG_FILE)) HELMFILE_SELECTOR ?= HELMFILE_EXTRA_ARGS ?= --allow-no-matching-release -.PHONY: template install sync apply destroy clean help +.PHONY: template test install sync apply destroy clean help .DEFAULT_GOAL := help template: clean @@ -22,6 +22,16 @@ template: clean $(if $(HELMFILE_SELECTOR),--selector $(HELMFILE_SELECTOR)) \ template --output-dir "$(OUTPUT_DIR)" +test: + @helm lint charts/nvcf-default-monitors + @helm lint charts/nvcf-observability-contract + @helm lint charts/nvcf-otel-collector + @helm lint ../../helm/function-autoscaler \ + --set functionautoscaler.image.registry=nvcr.io \ + --set functionautoscaler.image.repository=test/nvcf-function-autoscaler + @tests/profile-defaults.sh + @../self-managed/tests/observability-autoscaler.sh + install: @echo ">>> Installing observability stack" HELMFILE_ENV="$(HELMFILE_ENV)" \ @@ -55,6 +65,7 @@ help: @echo "" @echo "Targets:" @echo " template Generate Kubernetes manifests without applying" + @echo " test Validate profile release and monitor defaults" @echo " install Install the stack with helmfile sync" @echo " apply Apply changes with helmfile apply" @echo " destroy Remove stack releases" diff --git a/deploy/stacks/observability/README.md b/deploy/stacks/observability/README.md index d4d84a7ad..cef84aa56 100644 --- a/deploy/stacks/observability/README.md +++ b/deploy/stacks/observability/README.md @@ -1,128 +1,202 @@ # nvcf-observability-stack -Reusable Helmfile stack scaffold for self-hosted NVCF observability. It is -consumed by self-managed control-plane deployments and optional standalone -compute-plane deployments. +Reusable Helmfile stack for self-hosted NVCF observability. A cluster installs +this stack at most once and selects the targets with one value: + +```yaml +observability: + profile: control +``` The stack can own: -- Prometheus Operator CRDs required for `ServiceMonitor` and `PodMonitor` +- Prometheus Operator CRDs for `ServiceMonitor` and `PodMonitor` - OpenTelemetry Operator -- Control-plane OpenTelemetry Collector with Target Allocator support -- Read-only RBAC for monitor discovery and Kubernetes service discovery -- VictoriaMetrics for the bundled local TSDB -- Default NVCF monitor resources, starting with DCGM - -BYOO and NVCA collectors are separate optional features and default off. +- One OpenTelemetry Collector with Target Allocator support +- Read-only discovery RBAC +- VictoriaMetrics +- NVCF-owned default monitor resources -## Modes +## Profiles -Set `observability.mode` per consuming stack. +| Profile | Control targets | Compute targets | Result | +| --- | --- | --- | --- | +| `disabled` | No | No | Render no observability releases or resources. | +| `control` | Yes | No | Install one stack for control-plane metrics and the autoscaler backend contract. | +| `compute` | No | Yes | Install one stack for NVCA, DCGM, and worker metrics; resolve NVCA BYOO support on. | +| `all` | Yes | Yes | Install the union once; do not duplicate shared components. | -| Mode | Behavior | -| --- | --- | -| `install` | Install enabled shared observability infrastructure in this cluster. | -| `existing` | Assume shared infrastructure already exists, but allow default monitors to be rendered against it. | -| `disabled` | Render no observability runtime resources. Valid for compute-plane deployments that do not need NVCF-managed scrape coverage. | +The packaged reusable stack defaults to `disabled`. Its self-managed +control-plane example overlay defaults to `control`. A standalone compute-plane +consumer should select `compute`. A colocated deployment should install this +artifact once with `all`; it must not configure a second observability-stack +release from the compute-plane consumer. -For a single-cluster self-managed deployment, the consuming control-plane stack -should use `install` and the consuming compute-plane stack should use -`existing`. For standalone compute-plane clusters, the compute-plane stack -should use `install` only when NVCF-managed NVCA, DCGM, or worker metric -scraping is required. +The public configuration does not expose +`planes.control.enabled` or `planes.compute.enabled`. Those values are derived +internally from the profile. -## Component Gates +## Profile Defaults -Base defaults keep runtime resources disabled until a consuming stack wires the -chart repository, mirrored images, release artifact path, and environment -overlay. +Every enabled profile installs these shared components by default: -A single-cluster self-managed overlay should enable the control-plane path: +- Prometheus Operator CRDs +- OpenTelemetry Operator +- One OpenTelemetry Collector +- Target Allocator +- Discovery RBAC +- One VictoriaMetrics instance + +The profile-specific defaults are: + +| Default | `disabled` | `control` | `compute` | `all` | +| --- | --- | --- | --- | --- | +| Control-plane monitors | Off | On | Off | On | +| NVCA `ServiceMonitor` | Off | Off | On | On | +| DCGM `PodMonitor` | Off | Off | On | On | +| Worker `PodMonitor` | Off | Off | On | On | +| BYOO support | Off | Off | On | On | +| Autoscaler integration | Off | On | Off | On | +| NVCA collector | Off | Off | Off | Off | + +Control-plane monitoring covers State Metric Service, Invocation Service, gRPC +Proxy, and LLM API Gateway. Compute monitoring selects NVCA in `nvca-system`, +DCGM pods that carry NVCA's DCGM metrics label, and pods exposing a `metrics` +port in NVCA-managed workload namespaces. + +For `compute` and `all`, the resolved profile contract defaults BYOO support on. +The NVCA installer should map that value to its existing `BYOObservability` +feature gate. The feature gate enables the per-function BYOO collector path; it +does not create a second shared collector. This stack does not deploy NVCA. + +Every enabled profile publishes its resolved consumer defaults in the +`nvcf-observability-profile` ConfigMap in the contract namespace, `nvcf` by +default. That contract lets separately packaged consumers use the profile +result without reimplementing profile logic or installing this stack again. + +## Fine-Grained Overrides + +Profiles provide normal defaults. Advanced installations can override shared +component ownership with `install`, `existing`, or `disabled`. + +The fully expanded ownership configuration below is equivalent to setting only +`observability.profile: control`: ```yaml observability: + profile: control + components: + prometheusOperatorCrds: + mode: install + otelOperator: + mode: install + collector: + mode: install + targetAllocator: + mode: install + discoveryRbac: + mode: install + +metricsBackend: mode: install - namespace: monitoring - -prometheusOperatorCrds: - enabled: true - -opentelemetryOperator: - enabled: true - -controlPlaneCollector: - enabled: true + type: victoriaMetrics +``` -victoriaMetrics: - enabled: true +For example, a deployment that supplies its own operator and metrics backend +can override only those ownership decisions: -defaultMonitors: - enabled: true - dcgm: - enabled: true +```yaml +observability: + profile: control + components: + otelOperator: + mode: existing + +metricsBackend: + mode: existing + type: external + remoteWriteEndpoint: https://metrics.example.com/write + promqlEndpoint: https://metrics.example.com ``` -`environments/local.yaml` disables runtime resources so the scaffold can render -offline without a cluster or registry configuration. - -When `controlPlaneCollector.enabled` is true in `install` mode, the stack uses -the OpenTelemetry Operator and Prometheus Operator CRDs configured by values. -Enable both unless the deployment uses a customer-managed operator or CRD -contract, because the collector is an `OpenTelemetryCollector` CR and the Target -Allocator monitor path requires the monitor CRDs. +Supported component paths are: -The default namespace is `monitoring` so collector traffic matches NVCA's -current DCGM metrics NetworkPolicy. If a consuming stack changes the namespace, -it must also update the NVCA NetworkPolicy contract or provide an equivalent -reachability rule. +- `observability.components.prometheusOperatorCrds.mode` +- `observability.components.otelOperator.mode` +- `observability.components.collector.mode` +- `observability.components.targetAllocator.mode` +- `observability.components.discoveryRbac.mode` +- `metricsBackend.mode` -For EKS/ADOT-style environments where another operator owns the -OpenTelemetry CRDs, set: +`install` makes this stack the owner. `existing` skips installation and requires +a separate installer preflight to verify a compatible external component. +`disabled` does not install or use the component. Overrides that contradict +required dependencies fail during Helmfile rendering. Component overrides +cannot install resources under `profile: disabled`. -```yaml -opentelemetryOperator: - enabled: false +The bundled backend derives both endpoints from the VictoriaMetrics namespace: -prometheusOperatorCrds: - enabled: false +```text +remote write: http://vmsingle.monitoring.svc.cluster.local:8428/api/v1/write +PromQL: http://vmsingle.monitoring.svc.cluster.local:8428 ``` -The control-plane collector can still be configured to remote-write to a -customer-owned TSDB, and the autoscaler chart should read from that same TSDB. -When the bundled VictoriaMetrics release is enabled, the collector remote-write -endpoint must match the VictoriaMetrics namespace. +An external backend must provide both endpoints for `control` and `all`. -## Default Monitors +## Autoscaler Contract -Application charts own stable metrics contracts: service names or labels, pod -labels, port names, paths, and namespaces. This stack owns the default monitor -resources that select those contracts. +The autoscaler is a control-plane consumer, not a second observability stack. +It reads from the same `metricsBackend.promqlEndpoint` selected by the profile. +For `control` and `all`, the stack publishes that resolved contract as +`nvcf-observability-autoscaler` in the `nvcf` namespace: + +```yaml +data: + TIMESERIES_DB__TIMESERIES_DB_URL: http://vmsingle.monitoring.svc.cluster.local:8428 + TIMESERIES_DB__AUTH_MODE: none + TIMESERIES_DB__IGNORE_ENV: "true" +``` -The initial scaffold includes one concrete default monitor: +`autoscalerIntegration.namespace` sets the contract release namespace, and +`autoscalerIntegration.configMapName` overrides the autoscaler ConfigMap name. +For an external backend, `metricsBackend.authentication.mode` supports `none`, +`token`, and `mtls`. Token mode also requires `authnEndpoint`; mTLS mode +requires paths for the mounted client certificate and private key. -- DCGM exporter pods selected by `nvca.nvcf.nvidia.io/dcgm-metrics-present: "true"` on the `dcgm-metrics` named port +The self-managed Helmfile deploys the function autoscaler for `control` and +`all` and consumes this ConfigMap through the chart's external `envFrom` +reference. It also supplies the Cassandra and NVCF API endpoints needed by the +self-managed runtime. The autoscaler does not need another observability enable +flag, backend mode, or duplicated VictoriaMetrics URL. The `compute` and +`disabled` profiles do not deploy it. -Keep the DCGM monitor disabled until the consuming stack owns the NVCA chart -contract that guarantees the injected DCGM metrics port keeps the -`dcgm-metrics` name. +## Monitor Contracts -Default monitors carry `nvcf.nvidia.com/observability-target: "true"` so the Target -Allocator can select only NVCF-owned scrape targets by default. +Application charts own stable metrics endpoints, labels, named ports, paths, +and namespaces. This stack owns the default monitor resources that select those +contracts. -Add NVCA, State Metric Service, worker, or control-plane service monitors as -concrete templates only after their chart-owned scrape contracts are stable. +Default monitors carry +`nvcf.nvidia.com/observability-target: "true"` so the Target Allocator selects +only NVCF-owned scrape targets. -The control-plane collector cannot discover compute-plane-local scrape targets -in a different Kubernetes cluster. Decoupled compute clusters need their own -observability-stack instance, customer-owned observability, or no NVCF-managed -DCGM/NVCA/worker scrape coverage. +The default namespace is `monitoring`. If a deployment changes it, it must also +provide NetworkPolicy reachability for the collector. ## Local Rendering +The checked-in local environment is disabled by default: + ```sh make template HELMFILE_ENV=local ``` -`environments/local.yaml` is an opt-in example. Replace the chart repository, -image repository, and exporter endpoint with values for the target cluster -before enabling runtime components. +Change only `observability.profile` in the environment file to render a +complete enabled profile. Replace the example chart repository and image +repository before installing in a cluster. + +Run the Helmfile profile assertions and autoscaler chart render checks with: + +```sh +make test +``` diff --git a/deploy/stacks/observability/charts/nvcf-default-monitors/Chart.yaml b/deploy/stacks/observability/charts/nvcf-default-monitors/Chart.yaml index a91bc3850..65ba003c7 100644 --- a/deploy/stacks/observability/charts/nvcf-default-monitors/Chart.yaml +++ b/deploy/stacks/observability/charts/nvcf-default-monitors/Chart.yaml @@ -17,5 +17,5 @@ apiVersion: v2 name: nvcf-default-monitors description: Concrete default NVCF monitor resources for the observability stack type: application -version: 0.1.0 -appVersion: "0.1.0" +version: 0.2.0 +appVersion: "0.2.0" diff --git a/deploy/stacks/observability/charts/nvcf-default-monitors/templates/dcgm-podmonitor.yaml b/deploy/stacks/observability/charts/nvcf-default-monitors/templates/dcgm-podmonitor.yaml index 0ad0f33aa..563c4618f 100644 --- a/deploy/stacks/observability/charts/nvcf-default-monitors/templates/dcgm-podmonitor.yaml +++ b/deploy/stacks/observability/charts/nvcf-default-monitors/templates/dcgm-podmonitor.yaml @@ -15,7 +15,7 @@ See the License for the specific language governing permissions and limitations under the License. */}} -{{- if .Values.dcgm.enabled }} +{{- if and .Values.computePlane.enabled .Values.dcgm.enabled }} apiVersion: monitoring.coreos.com/v1 kind: PodMonitor metadata: diff --git a/deploy/stacks/observability/charts/nvcf-default-monitors/templates/nvca-servicemonitor.yaml b/deploy/stacks/observability/charts/nvcf-default-monitors/templates/nvca-servicemonitor.yaml new file mode 100644 index 000000000..d51138b26 --- /dev/null +++ b/deploy/stacks/observability/charts/nvcf-default-monitors/templates/nvca-servicemonitor.yaml @@ -0,0 +1,39 @@ +{{/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.computePlane.enabled .Values.nvca.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "nvcf-default-monitors.name" . }}-nvca + namespace: {{ .Release.Namespace }} + labels: + {{- include "nvcf-default-monitors.labels" . | nindent 4 }} + {{- with .Values.nvca.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + namespaceSelector: + {{- toYaml .Values.nvca.namespaceSelector | nindent 4 }} + selector: + matchLabels: + {{- toYaml .Values.nvca.selector | nindent 6 }} + endpoints: + - interval: {{ .Values.nvca.interval | quote }} + path: {{ .Values.nvca.path | quote }} + port: {{ .Values.nvca.port | quote }} +{{- end }} diff --git a/deploy/stacks/observability/charts/nvcf-default-monitors/templates/worker-podmonitor.yaml b/deploy/stacks/observability/charts/nvcf-default-monitors/templates/worker-podmonitor.yaml new file mode 100644 index 000000000..557feeddb --- /dev/null +++ b/deploy/stacks/observability/charts/nvcf-default-monitors/templates/worker-podmonitor.yaml @@ -0,0 +1,38 @@ +{{/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.computePlane.enabled .Values.worker.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: PodMonitor +metadata: + name: {{ include "nvcf-default-monitors.name" . }}-worker + namespace: {{ .Release.Namespace }} + labels: + {{- include "nvcf-default-monitors.labels" . | nindent 4 }} + {{- with .Values.worker.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + namespaceSelector: + {{- toYaml .Values.worker.namespaceSelector | nindent 4 }} + selector: + matchLabels: {{ .Values.worker.selector | toJson }} + podMetricsEndpoints: + - interval: {{ .Values.worker.interval | quote }} + path: {{ .Values.worker.path | quote }} + port: {{ .Values.worker.port | quote }} +{{- end }} diff --git a/deploy/stacks/observability/charts/nvcf-default-monitors/values.yaml b/deploy/stacks/observability/charts/nvcf-default-monitors/values.yaml index 1605cd7ee..81e287285 100644 --- a/deploy/stacks/observability/charts/nvcf-default-monitors/values.yaml +++ b/deploy/stacks/observability/charts/nvcf-default-monitors/values.yaml @@ -28,8 +28,8 @@ dcgm: port: dcgm-metrics labels: {} -# Control-plane ServiceMonitors, owned by the obs stack. Each renders one -# ServiceMonitor that picks up the observability-target label via commonLabels. +# These plane gates are internal inputs derived from observability.profile by +# the parent stack. They are not public user-facing switches. controlPlane: enabled: false interval: 30s @@ -49,22 +49,38 @@ controlPlane: enabled: true selector: app.kubernetes.io/instance: grpc-proxy - - name: api - enabled: true - selector: - app.kubernetes.io/name: helm-nvcf-api - - name: sis - enabled: true - namespaces: - - sis - selector: - app.kubernetes.io/name: helm-nvcf-sis - name: llm-api-gateway enabled: true selector: app.kubernetes.io/instance: llm-api-gateway - # Off until the Invocation Service exporter is enabled. - name: invocation-service - enabled: false + enabled: true selector: app.kubernetes.io/instance: invocation-service + +computePlane: + enabled: false + +nvca: + enabled: true + namespaceSelector: + matchNames: + - nvca-system + interval: 30s + path: /metrics + port: nvca + selector: + app.kubernetes.io/name: nvca + labels: {} + +worker: + enabled: true + namespaceSelector: + matchExpressions: + - key: nvca.nvcf.nvidia.io/workload-instance-type + operator: Exists + interval: 30s + path: /metrics + port: metrics + selector: {} + labels: {} diff --git a/deploy/stacks/observability/charts/nvcf-observability-contract/Chart.yaml b/deploy/stacks/observability/charts/nvcf-observability-contract/Chart.yaml new file mode 100644 index 000000000..deea259b2 --- /dev/null +++ b/deploy/stacks/observability/charts/nvcf-observability-contract/Chart.yaml @@ -0,0 +1,21 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v2 +name: nvcf-observability-contract +description: Resolved observability settings for NVCF stack consumers +type: application +version: 0.1.0 +appVersion: "0.1.0" diff --git a/deploy/stacks/observability/charts/nvcf-observability-contract/templates/autoscaler-configmap.yaml b/deploy/stacks/observability/charts/nvcf-observability-contract/templates/autoscaler-configmap.yaml new file mode 100644 index 000000000..5de41d5df --- /dev/null +++ b/deploy/stacks/observability/charts/nvcf-observability-contract/templates/autoscaler-configmap.yaml @@ -0,0 +1,42 @@ +{{/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.autoscaler.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Values.autoscaler.configMapName }} + namespace: {{ .Values.autoscaler.namespace }} + labels: + app.kubernetes.io/name: nvcf-observability-contract + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + nvcf.nvidia.com/observability-profile: {{ .Values.profile | quote }} +data: + TIMESERIES_DB__TIMESERIES_DB_URL: {{ required "autoscaler.promqlEndpoint is required for control and all profiles" .Values.autoscaler.promqlEndpoint | quote }} + TIMESERIES_DB__AUTH_MODE: {{ .Values.autoscaler.authMode | quote }} + TIMESERIES_DB__IGNORE_ENV: {{ .Values.autoscaler.ignoreEnv | quote }} + {{- with .Values.autoscaler.authnEndpoint }} + TIMESERIES_DB__AUTHN_URL: {{ . | quote }} + {{- end }} + {{- with .Values.autoscaler.clientCertificatePath }} + TIMESERIES_DB__CLIENT_CERTIFICATE_PATH: {{ . | quote }} + {{- end }} + {{- with .Values.autoscaler.clientPrivateKeyPath }} + TIMESERIES_DB__CLIENT_PRIVATE_KEY_PATH: {{ . | quote }} + {{- end }} +{{- end }} diff --git a/deploy/stacks/observability/charts/nvcf-observability-contract/templates/profile-configmap.yaml b/deploy/stacks/observability/charts/nvcf-observability-contract/templates/profile-configmap.yaml new file mode 100644 index 000000000..cb0f41309 --- /dev/null +++ b/deploy/stacks/observability/charts/nvcf-observability-contract/templates/profile-configmap.yaml @@ -0,0 +1,36 @@ +{{/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if ne .Values.profile "disabled" }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: nvcf-observability-profile + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/name: nvcf-observability-contract + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + nvcf.nvidia.com/observability-profile: {{ .Values.profile | quote }} +data: + profile: {{ .Values.profile | quote }} + controlTargetsEnabled: {{ .Values.controlTargetsEnabled | quote }} + computeTargetsEnabled: {{ .Values.computeTargetsEnabled | quote }} + byooSupportEnabled: {{ .Values.byooSupportEnabled | quote }} + nvcaCollectorEnabled: {{ .Values.nvcaCollectorEnabled | quote }} + autoscalerIntegrationEnabled: {{ .Values.autoscaler.enabled | quote }} +{{- end }} diff --git a/deploy/stacks/observability/charts/nvcf-observability-contract/values.yaml b/deploy/stacks/observability/charts/nvcf-observability-contract/values.yaml new file mode 100644 index 000000000..a7455fc30 --- /dev/null +++ b/deploy/stacks/observability/charts/nvcf-observability-contract/values.yaml @@ -0,0 +1,32 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +profile: disabled + +controlTargetsEnabled: false +computeTargetsEnabled: false +byooSupportEnabled: false +nvcaCollectorEnabled: false + +autoscaler: + enabled: false + namespace: nvcf + configMapName: nvcf-observability-autoscaler + promqlEndpoint: "" + authMode: none + authnEndpoint: "" + clientCertificatePath: "" + clientPrivateKeyPath: "" + ignoreEnv: true diff --git a/deploy/stacks/observability/charts/nvcf-otel-collector/Chart.yaml b/deploy/stacks/observability/charts/nvcf-otel-collector/Chart.yaml index ad4e2365b..73e44a256 100644 --- a/deploy/stacks/observability/charts/nvcf-otel-collector/Chart.yaml +++ b/deploy/stacks/observability/charts/nvcf-otel-collector/Chart.yaml @@ -15,7 +15,7 @@ apiVersion: v2 name: nvcf-otel-collector -description: Control-plane OpenTelemetryCollector with Target Allocator support for NVCF scrape discovery +description: OpenTelemetryCollector with Target Allocator support for NVCF scrape discovery type: application -version: 0.1.0 -appVersion: "0.1.0" +version: 0.2.0 +appVersion: "0.2.0" diff --git a/deploy/stacks/observability/charts/nvcf-otel-collector/values.yaml b/deploy/stacks/observability/charts/nvcf-otel-collector/values.yaml index 517e42e7f..d0f0a807e 100644 --- a/deploy/stacks/observability/charts/nvcf-otel-collector/values.yaml +++ b/deploy/stacks/observability/charts/nvcf-otel-collector/values.yaml @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -collectorName: nvcf-control-plane +collectorName: nvcf-observability image: ghcr.io/open-telemetry/opentelemetry-collector-releases/opentelemetry-collector-contrib:0.129.1 imagePullSecrets: [] diff --git a/deploy/stacks/observability/environments/base.yaml b/deploy/stacks/observability/environments/base.yaml index 11f3b91a5..b30aeeab5 100644 --- a/deploy/stacks/observability/environments/base.yaml +++ b/deploy/stacks/observability/environments/base.yaml @@ -10,11 +10,31 @@ global: imagePullSecrets: [] observability: - mode: disabled + profile: disabled namespace: monitoring +metricsBackend: + type: victoriaMetrics + # Used only with mode: existing. The bundled VictoriaMetrics endpoints are + # derived automatically when the selected profile installs the backend. + remoteWriteEndpoint: "" + promqlEndpoint: "" + authentication: + # Autoscaler PromQL authentication. Supported modes are none, token, and + # mtls. The bundled VictoriaMetrics backend always resolves to none. + mode: none + authnEndpoint: "" + clientCertificatePath: "" + clientPrivateKeyPath: "" + +autoscalerIntegration: + namespace: nvcf + configMapName: nvcf-observability-autoscaler + # Self-hosted control-plane metrics do not require the hosted TSDB's + # environment label or authentication flow. + ignoreEnv: true + prometheusOperatorCrds: - enabled: false version: "30.0.0" values: crds: @@ -40,7 +60,6 @@ prometheusOperatorCrds: enabled: false opentelemetryOperator: - enabled: false version: "0.114.1" admissionWebhooks: certManager: @@ -52,14 +71,10 @@ opentelemetryOperator: repository: ghcr.io/open-telemetry/opentelemetry-collector-releases/opentelemetry-collector-contrib tag: "0.129.1" -controlPlaneCollector: - enabled: false - collectorName: nvcf-control-plane +collector: + collectorName: nvcf-observability image: ghcr.io/open-telemetry/opentelemetry-collector-releases/opentelemetry-collector-contrib:0.129.1 - targetAllocatorRbac: - create: true targetAllocator: - enabled: true allocationStrategy: consistent-hashing prometheusCR: enabled: true @@ -80,7 +95,8 @@ controlPlaneCollector: batch: {} exporters: prometheusremotewrite: - endpoint: http://vmsingle.monitoring.svc.cluster.local:8428/api/v1/write + # Filled from metricsBackend after profile defaults are resolved. + endpoint: "" service: pipelines: metrics: @@ -92,7 +108,6 @@ controlPlaneCollector: - prometheusremotewrite victoriaMetrics: - enabled: false version: "0.38.0" server: retentionPeriod: "1" @@ -108,23 +123,62 @@ victoriaMetrics: cpu: "1" memory: 1Gi serviceMonitor: - enabled: false + enabled: true defaultMonitors: - enabled: false commonLabels: nvcf.nvidia.com/observability-target: "true" + controlPlane: + interval: 30s + path: /metrics + port: metrics + namespaces: + - nvcf + services: + - name: state-metrics + enabled: true + selector: + app.kubernetes.io/instance: state-metrics + app.kubernetes.io/name: helm-nvcf-state-metrics + - name: invocation-service + enabled: true + selector: + app.kubernetes.io/instance: invocation-service + - name: grpc-proxy + enabled: true + selector: + app.kubernetes.io/instance: grpc-proxy + - name: llm-api-gateway + enabled: true + selector: + app.kubernetes.io/instance: llm-api-gateway + nvca: + enabled: true + namespaceSelector: + matchNames: + - nvca-system + interval: 30s + path: /metrics + port: nvca + selector: + app.kubernetes.io/name: nvca + labels: {} dcgm: - enabled: false + enabled: true namespaceSelector: any: true interval: 30s path: /metrics port: dcgm-metrics labels: {} - -byooCollector: - enabled: false - -nvcaCollector: - enabled: false + worker: + enabled: true + namespaceSelector: + matchExpressions: + - key: nvca.nvcf.nvidia.io/workload-instance-type + operator: Exists + interval: 30s + path: /metrics + port: metrics + selector: {} + labels: {} diff --git a/deploy/stacks/observability/environments/local.yaml b/deploy/stacks/observability/environments/local.yaml index f9f4e5bb0..a2cb5584e 100644 --- a/deploy/stacks/observability/environments/local.yaml +++ b/deploy/stacks/observability/environments/local.yaml @@ -1,21 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -# Local example. Runtime components remain off until explicitly enabled. +# Local example. Change this one value to control, compute, or all to render the +# corresponding complete default stack. observability: - mode: disabled - -prometheusOperatorCrds: - enabled: false - -opentelemetryOperator: - enabled: false - -controlPlaneCollector: - enabled: false - -victoriaMetrics: - enabled: false - -defaultMonitors: - enabled: false + profile: disabled diff --git a/deploy/stacks/observability/helmfile.d/01-observability.yaml.gotmpl b/deploy/stacks/observability/helmfile.d/01-observability.yaml.gotmpl index 976f078de..c8b712114 100644 --- a/deploy/stacks/observability/helmfile.d/01-observability.yaml.gotmpl +++ b/deploy/stacks/observability/helmfile.d/01-observability.yaml.gotmpl @@ -6,24 +6,119 @@ environments: --- -{{- $mode := dig "observability" "mode" "disabled" .Values }} +{{- $profile := dig "observability" "profile" "disabled" .Values }} +{{- if not (has $profile (list "disabled" "control" "compute" "all")) }} +{{- fail (printf "observability.profile must be disabled, control, compute, or all, got %q" $profile) }} +{{- end }} +{{- $profileEnabled := ne $profile "disabled" }} +{{- $controlEnabled := or (eq $profile "control") (eq $profile "all") }} +{{- $computeEnabled := or (eq $profile "compute") (eq $profile "all") }} +{{- $defaultComponentMode := ternary "install" "disabled" $profileEnabled }} +{{- $prometheusOperatorCrdsMode := dig "observability" "components" "prometheusOperatorCrds" "mode" $defaultComponentMode .Values }} +{{- $opentelemetryOperatorMode := dig "observability" "components" "otelOperator" "mode" $defaultComponentMode .Values }} +{{- $collectorMode := dig "observability" "components" "collector" "mode" $defaultComponentMode .Values }} +{{- $targetAllocatorMode := dig "observability" "components" "targetAllocator" "mode" $defaultComponentMode .Values }} +{{- $discoveryRbacMode := dig "observability" "components" "discoveryRbac" "mode" $defaultComponentMode .Values }} +{{- $metricsBackendMode := dig "metricsBackend" "mode" $defaultComponentMode .Values }} +{{- $componentModes := dict + "observability.components.prometheusOperatorCrds.mode" $prometheusOperatorCrdsMode + "observability.components.otelOperator.mode" $opentelemetryOperatorMode + "observability.components.collector.mode" $collectorMode + "observability.components.targetAllocator.mode" $targetAllocatorMode + "observability.components.discoveryRbac.mode" $discoveryRbacMode + "metricsBackend.mode" $metricsBackendMode }} +{{- range $name, $mode := $componentModes }} {{- if not (has $mode (list "install" "existing" "disabled")) }} -{{- fail (printf "observability.mode must be install, existing, or disabled, got %q" $mode) }} +{{- fail (printf "%s must be install, existing, or disabled, got %q" $name $mode) }} +{{- end }} +{{- if and (not $profileEnabled) (ne $mode "disabled") }} +{{- fail (printf "%s cannot be %q when observability.profile=disabled" $name $mode) }} +{{- end }} +{{- end }} +{{- if and (eq $collectorMode "install") (eq $opentelemetryOperatorMode "disabled") }} +{{- fail "observability.components.otelOperator.mode cannot be disabled when the collector is installed; use existing for a customer-managed operator" }} +{{- end }} +{{- if and (eq $collectorMode "install") (eq $prometheusOperatorCrdsMode "disabled") (eq $targetAllocatorMode "install") }} +{{- fail "observability.components.prometheusOperatorCrds.mode cannot be disabled when the collector installs the Target Allocator" }} +{{- end }} +{{- if and $profileEnabled (eq $prometheusOperatorCrdsMode "disabled") }} +{{- fail "observability.components.prometheusOperatorCrds.mode cannot be disabled for an enabled profile because default monitors require the APIs; use existing for customer-managed CRDs" }} +{{- end }} +{{- if and (eq $collectorMode "install") (eq $metricsBackendMode "disabled") }} +{{- fail "metricsBackend.mode cannot be disabled when the collector is installed; use existing for an external backend" }} +{{- end }} +{{- if and $controlEnabled (eq $metricsBackendMode "disabled") }} +{{- fail "metricsBackend.mode cannot be disabled for control or all because the autoscaler requires a PromQL endpoint" }} +{{- end }} +{{- if and (eq $collectorMode "install") (eq $targetAllocatorMode "existing") }} +{{- fail "observability.components.targetAllocator.mode=existing requires observability.components.collector.mode=existing; use install or disabled with a stack-owned collector" }} +{{- end }} +{{- if and (ne $collectorMode "install") (eq $targetAllocatorMode "install") }} +{{- fail "observability.components.targetAllocator.mode=install requires observability.components.collector.mode=install" }} +{{- end }} +{{- if and (ne $collectorMode "install") (eq $discoveryRbacMode "install") }} +{{- fail "observability.components.discoveryRbac.mode=install requires observability.components.collector.mode=install" }} +{{- end }} +{{- if and (eq $targetAllocatorMode "install") (eq $discoveryRbacMode "disabled") }} +{{- fail "observability.components.discoveryRbac.mode cannot be disabled when the Target Allocator is installed; use existing for externally managed RBAC" }} {{- end }} -{{- $installMode := eq $mode "install" }} -{{- $existingMode := eq $mode "existing" }} {{- $observabilityNamespace := dig "observability" "namespace" "monitoring" .Values }} -{{- $controlPlaneCollectorEnabled := and $installMode (dig "controlPlaneCollector" "enabled" false .Values) }} -{{- $defaultMonitorsEnabled := and (or $installMode $existingMode) (dig "defaultMonitors" "enabled" false .Values) }} -{{- $prometheusOperatorCrdsEnabled := and $installMode (dig "prometheusOperatorCrds" "enabled" false .Values) }} -{{- $opentelemetryOperatorEnabled := and $installMode (dig "opentelemetryOperator" "enabled" false .Values) }} -{{- $victoriaMetricsEnabled := and $installMode (dig "victoriaMetrics" "enabled" false .Values) }} +{{- $prometheusOperatorCrdsEnabled := eq $prometheusOperatorCrdsMode "install" }} +{{- $opentelemetryOperatorEnabled := eq $opentelemetryOperatorMode "install" }} +{{- $collectorEnabled := eq $collectorMode "install" }} +{{- $targetAllocatorEnabled := eq $targetAllocatorMode "install" }} +{{- $discoveryRbacEnabled := eq $discoveryRbacMode "install" }} +{{- $victoriaMetricsEnabled := eq $metricsBackendMode "install" }} +{{- $defaultMonitorsEnabled := $profileEnabled }} +{{- $autoscalerIntegrationEnabled := $controlEnabled }} +{{- $contractNamespace := dig "autoscalerIntegration" "namespace" "nvcf" .Values }} {{- $victoriaMetricsNamespace := dig "victoriaMetrics" "namespace" $observabilityNamespace .Values }} {{- $bundledRemoteWriteEndpoint := printf "http://vmsingle.%s.svc.cluster.local:8428/api/v1/write" $victoriaMetricsNamespace }} -{{- $configuredRemoteWriteEndpoint := dig "controlPlaneCollector" "config" "exporters" "prometheusremotewrite" "endpoint" "" .Values }} -{{- if and $controlPlaneCollectorEnabled $victoriaMetricsEnabled (ne $configuredRemoteWriteEndpoint $bundledRemoteWriteEndpoint) }} -{{- fail (printf "controlPlaneCollector.config.exporters.prometheusremotewrite.endpoint must be %q when victoriaMetrics.enabled=true, got %q" $bundledRemoteWriteEndpoint $configuredRemoteWriteEndpoint) }} +{{- $bundledPromqlEndpoint := printf "http://vmsingle.%s.svc.cluster.local:8428" $victoriaMetricsNamespace }} +{{- $metricsBackendType := dig "metricsBackend" "type" "victoriaMetrics" .Values }} +{{- $metricsBackendAuthMode := dig "metricsBackend" "authentication" "mode" "none" .Values }} +{{- if not (has $metricsBackendAuthMode (list "none" "token" "mtls")) }} +{{- fail (printf "metricsBackend.authentication.mode must be none, token, or mtls, got %q" $metricsBackendAuthMode) }} +{{- end }} +{{- if and $victoriaMetricsEnabled (ne $metricsBackendType "victoriaMetrics") }} +{{- fail (printf "metricsBackend.type must be victoriaMetrics when metricsBackend.mode=install, got %q" $metricsBackendType) }} +{{- end }} +{{- if and $victoriaMetricsEnabled (ne $metricsBackendAuthMode "none") }} +{{- fail "metricsBackend.authentication.mode must be none when metricsBackend.mode=install" }} {{- end }} +{{- if and (eq $metricsBackendMode "existing") (ne $metricsBackendType "external") }} +{{- fail (printf "metricsBackend.type must be external when metricsBackend.mode=existing, got %q" $metricsBackendType) }} +{{- end }} +{{- $remoteWriteEndpoint := ternary $bundledRemoteWriteEndpoint (dig "metricsBackend" "remoteWriteEndpoint" "" .Values) $victoriaMetricsEnabled }} +{{- $promqlEndpoint := ternary $bundledPromqlEndpoint (dig "metricsBackend" "promqlEndpoint" "" .Values) $victoriaMetricsEnabled }} +{{- if and (eq $metricsBackendMode "existing") (empty $remoteWriteEndpoint) }} +{{- fail "metricsBackend.remoteWriteEndpoint is required when metricsBackend.mode=existing" }} +{{- end }} +{{- if and $controlEnabled (eq $metricsBackendMode "existing") (empty $promqlEndpoint) }} +{{- fail "metricsBackend.promqlEndpoint is required for control and all profiles when metricsBackend.mode=existing" }} +{{- end }} +{{- $metricsBackendAuthnEndpoint := dig "metricsBackend" "authentication" "authnEndpoint" "" .Values }} +{{- $metricsBackendClientCertificatePath := dig "metricsBackend" "authentication" "clientCertificatePath" "" .Values }} +{{- $metricsBackendClientPrivateKeyPath := dig "metricsBackend" "authentication" "clientPrivateKeyPath" "" .Values }} +{{- if and $controlEnabled (eq $metricsBackendAuthMode "token") (empty $metricsBackendAuthnEndpoint) }} +{{- fail "metricsBackend.authentication.authnEndpoint is required for token authentication" }} +{{- end }} +{{- if and $controlEnabled (eq $metricsBackendAuthMode "mtls") (or (empty $metricsBackendClientCertificatePath) (empty $metricsBackendClientPrivateKeyPath)) }} +{{- fail "metricsBackend.authentication.clientCertificatePath and clientPrivateKeyPath are required for mtls authentication" }} +{{- end }} +{{- $collectorConfig := deepCopy (dig "collector" "config" dict .Values) }} +{{- $collectorExporters := deepCopy (dig "exporters" dict $collectorConfig) }} +{{- $prometheusRemoteWrite := deepCopy (dig "prometheusremotewrite" dict $collectorExporters) }} +{{- $_ := set $prometheusRemoteWrite "endpoint" $remoteWriteEndpoint }} +{{- $_ := set $collectorExporters "prometheusremotewrite" $prometheusRemoteWrite }} +{{- $_ := set $collectorConfig "exporters" $collectorExporters }} +{{- $defaultMonitors := deepCopy (dig "defaultMonitors" dict .Values) }} +{{- $controlPlaneMonitors := deepCopy (dig "controlPlane" dict $defaultMonitors) }} +{{- $_ := set $controlPlaneMonitors "enabled" $controlEnabled }} +{{- $_ := set $defaultMonitors "controlPlane" $controlPlaneMonitors }} +{{- $computePlaneMonitors := deepCopy (dig "computePlane" dict $defaultMonitors) }} +{{- $_ := set $computePlaneMonitors "enabled" $computeEnabled }} +{{- $_ := set $defaultMonitors "computePlane" $computePlaneMonitors }} repositories: - name: nvcf @@ -96,10 +191,10 @@ releases: release-group: observability {{- end }} - {{- if $controlPlaneCollectorEnabled }} - - name: control-plane-collector + {{- if $collectorEnabled }} + - name: otel-collector chart: ../charts/nvcf-otel-collector - namespace: {{ dig "controlPlaneCollector" "namespace" $observabilityNamespace .Values }} + namespace: {{ dig "collector" "namespace" $observabilityNamespace .Values }} {{- if or $opentelemetryOperatorEnabled $prometheusOperatorCrdsEnabled $victoriaMetricsEnabled }} needs: {{- if $opentelemetryOperatorEnabled }} @@ -113,18 +208,20 @@ releases: {{- end }} {{- end }} values: - - collectorName: {{ dig "controlPlaneCollector" "collectorName" "nvcf-control-plane" .Values | quote }} - image: {{ dig "controlPlaneCollector" "image" "" .Values | quote }} + - collectorName: {{ dig "collector" "collectorName" "nvcf-observability" .Values | quote }} + image: {{ dig "collector" "image" "" .Values | quote }} {{- if .Values.global.imagePullSecrets }} imagePullSecrets: {{- toYaml .Values.global.imagePullSecrets | nindent 10 }} {{- end }} targetAllocatorRbac: - create: {{ dig "controlPlaneCollector" "targetAllocatorRbac" "create" true .Values }} + create: {{ $discoveryRbacEnabled }} targetAllocator: - {{- toYaml (dig "controlPlaneCollector" "targetAllocator" dict .Values) | nindent 10 }} + {{- $targetAllocator := deepCopy (dig "collector" "targetAllocator" dict .Values) }} + {{- $_ := set $targetAllocator "enabled" $targetAllocatorEnabled }} + {{- toYaml $targetAllocator | nindent 10 }} config: - {{- toYaml (dig "controlPlaneCollector" "config" dict .Values) | nindent 10 }} + {{- toYaml $collectorConfig | nindent 10 }} labels: release-group: observability {{- end }} @@ -138,7 +235,35 @@ releases: - {{ dig "prometheusOperatorCrds" "namespace" $observabilityNamespace .Values }}/prometheus-operator-crds {{- end }} values: - - {{- toYaml (dig "defaultMonitors" dict .Values) | nindent 8 }} + - {{- toYaml $defaultMonitors | nindent 8 }} + labels: + release-group: observability + {{- end }} + + {{- if $profileEnabled }} + - name: observability-contract + chart: ../charts/nvcf-observability-contract + namespace: {{ $contractNamespace }} + {{- if $victoriaMetricsEnabled }} + needs: + - {{ $victoriaMetricsNamespace }}/victoria-metrics + {{- end }} + values: + - profile: {{ $profile | quote }} + controlTargetsEnabled: {{ $controlEnabled }} + computeTargetsEnabled: {{ $computeEnabled }} + byooSupportEnabled: {{ $computeEnabled }} + nvcaCollectorEnabled: false + autoscaler: + enabled: {{ $autoscalerIntegrationEnabled }} + namespace: {{ dig "autoscalerIntegration" "namespace" "nvcf" .Values | quote }} + configMapName: {{ dig "autoscalerIntegration" "configMapName" "nvcf-observability-autoscaler" .Values | quote }} + promqlEndpoint: {{ $promqlEndpoint | quote }} + authMode: {{ $metricsBackendAuthMode | quote }} + authnEndpoint: {{ $metricsBackendAuthnEndpoint | quote }} + clientCertificatePath: {{ $metricsBackendClientCertificatePath | quote }} + clientPrivateKeyPath: {{ $metricsBackendClientPrivateKeyPath | quote }} + ignoreEnv: {{ dig "autoscalerIntegration" "ignoreEnv" true .Values }} labels: release-group: observability {{- end }} diff --git a/deploy/stacks/observability/tests/profile-defaults.sh b/deploy/stacks/observability/tests/profile-defaults.sh new file mode 100755 index 000000000..7a7a8dce7 --- /dev/null +++ b/deploy/stacks/observability/tests/profile-defaults.sh @@ -0,0 +1,229 @@ +#!/usr/bin/env bash +set -euo pipefail + +stack_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +work_dir="$(mktemp -d)" +trap 'rm -rf "$work_dir"' EXIT + +fail() { + echo "profile-defaults: $*" >&2 + exit 1 +} + +assert_contract_value() { + local manifests="$1" + local profile="$2" + local key="$3" + local value="$4" + + grep -q "$key: \"$value\"" $manifests || + fail "$profile profile did not resolve $key to $value" +} + +profile_releases() { + local profile="$1" + HELMFILE_ENV=local helmfile \ + --file "$stack_dir/helmfile.d" \ + --environment default \ + --allow-no-matching-release \ + --state-values-set "observability.profile=$profile" \ + list 2>/dev/null | + awk 'NR > 1 {print $1}' | + sort +} + +render_monitors() { + local profile="$1" + local output_dir="$work_dir/$profile" + + HELMFILE_ENV=local helmfile \ + --file "$stack_dir/helmfile.d" \ + --environment default \ + --state-values-set "observability.profile=$profile" \ + --selector name=default-monitors \ + template --output-dir "$output_dir" >/dev/null +} + +enabled_releases="$( + cat <<'EOF' +default-monitors +observability-contract +opentelemetry-operator +otel-collector +prometheus-operator-crds +victoria-metrics +EOF +)" + +test -z "$(profile_releases disabled)" || + fail "disabled profile rendered releases" + +for profile in control compute all; do + test "$(profile_releases "$profile")" = "$enabled_releases" || + fail "$profile profile did not render the enabled release set exactly once" +done + +for profile in control compute all; do + render_monitors "$profile" +done + +control_manifests="$(find "$work_dir/control" -type f -name '*.yaml' -print)" +compute_manifests="$(find "$work_dir/compute" -type f -name '*.yaml' -print)" +all_manifests="$(find "$work_dir/all" -type f -name '*.yaml' -print)" + +grep -q 'nvcf-default-monitors-state-metrics' $control_manifests || + fail "control profile is missing control-plane monitors" +if grep -q 'nvcf-default-monitors-nvca' $control_manifests; then + fail "control profile rendered compute-plane monitors" +fi + +for monitor in nvca dcgm worker; do + grep -q "nvcf-default-monitors-$monitor" $compute_manifests || + fail "compute profile is missing $monitor monitor" + grep -q "nvcf-default-monitors-$monitor" $all_manifests || + fail "all profile is missing $monitor monitor" +done + +if grep -q 'nvcf-default-monitors-state-metrics' $compute_manifests; then + fail "compute profile rendered control-plane monitors" +fi +grep -q 'nvcf-default-monitors-state-metrics' $all_manifests || + fail "all profile is missing control-plane monitors" + +for profile in control compute all; do + HELMFILE_ENV=local helmfile \ + --file "$stack_dir/helmfile.d" \ + --environment default \ + --state-values-set "observability.profile=$profile" \ + --selector name=observability-contract \ + template --output-dir "$work_dir/$profile-contract" >/dev/null +done + +control_contract="$(find "$work_dir/control-contract" -type f -name '*.yaml' -print)" +compute_contract="$(find "$work_dir/compute-contract" -type f -name '*.yaml' -print)" +all_contract="$(find "$work_dir/all-contract" -type f -name '*.yaml' -print)" + +assert_contract_value "$control_contract" control controlTargetsEnabled true +assert_contract_value "$control_contract" control computeTargetsEnabled false +assert_contract_value "$control_contract" control byooSupportEnabled false +assert_contract_value "$control_contract" control nvcaCollectorEnabled false +assert_contract_value "$control_contract" control autoscalerIntegrationEnabled true + +assert_contract_value "$compute_contract" compute controlTargetsEnabled false +assert_contract_value "$compute_contract" compute computeTargetsEnabled true +assert_contract_value "$compute_contract" compute byooSupportEnabled true +assert_contract_value "$compute_contract" compute nvcaCollectorEnabled false +assert_contract_value "$compute_contract" compute autoscalerIntegrationEnabled false +if grep -q 'TIMESERIES_DB__TIMESERIES_DB_URL' $compute_contract; then + fail "compute profile published an autoscaler contract" +fi + +assert_contract_value "$all_contract" all controlTargetsEnabled true +assert_contract_value "$all_contract" all computeTargetsEnabled true +assert_contract_value "$all_contract" all byooSupportEnabled true +assert_contract_value "$all_contract" all nvcaCollectorEnabled false +assert_contract_value "$all_contract" all autoscalerIntegrationEnabled true + +HELMFILE_ENV=local helmfile \ + --file "$stack_dir/helmfile.d" \ + --environment default \ + --state-values-set observability.profile=control \ + --selector name=observability-contract \ + template --output-dir "$work_dir/bundled-contract" >/dev/null + +bundled_contract="$(find "$work_dir/bundled-contract" -type f -name '*.yaml' -print)" +grep -q 'TIMESERIES_DB__TIMESERIES_DB_URL: "http://vmsingle.monitoring.svc.cluster.local:8428"' \ + $bundled_contract || + fail "control profile did not publish the bundled PromQL endpoint" +grep -q 'TIMESERIES_DB__AUTH_MODE: "none"' $bundled_contract || + fail "control profile did not publish the bundled backend auth mode" + +HELMFILE_ENV=local helmfile \ + --file "$stack_dir/helmfile.d" \ + --environment default \ + --state-values-set observability.profile=control \ + --state-values-set observability.components.otelOperator.mode=existing \ + --state-values-set metricsBackend.mode=existing \ + --state-values-set metricsBackend.type=external \ + --state-values-set-string metricsBackend.remoteWriteEndpoint=https://metrics.example.com/write \ + --state-values-set-string metricsBackend.promqlEndpoint=https://metrics.example.com \ + --selector name=otel-collector \ + template --output-dir "$work_dir/external" >/dev/null + +external_manifests="$(find "$work_dir/external" -type f -name '*.yaml' -print)" +grep -q 'endpoint: https://metrics.example.com/write' $external_manifests || + fail "external remote-write endpoint was not applied to the collector" + +HELMFILE_ENV=local helmfile \ + --file "$stack_dir/helmfile.d" \ + --environment default \ + --state-values-set observability.profile=control \ + --state-values-set observability.components.otelOperator.mode=existing \ + --state-values-set metricsBackend.mode=existing \ + --state-values-set metricsBackend.type=external \ + --state-values-set-string metricsBackend.remoteWriteEndpoint=https://metrics.example.com/write \ + --state-values-set-string metricsBackend.promqlEndpoint=https://metrics.example.com \ + --selector name=observability-contract \ + template --output-dir "$work_dir/external-contract" >/dev/null + +external_contract="$(find "$work_dir/external-contract" -type f -name '*.yaml' -print)" +grep -q 'TIMESERIES_DB__TIMESERIES_DB_URL: "https://metrics.example.com"' \ + $external_contract || + fail "external PromQL endpoint was not published for the autoscaler" + +HELMFILE_ENV=local helmfile \ + --file "$stack_dir/helmfile.d" \ + --environment default \ + --state-values-set observability.profile=control \ + --state-values-set observability.components.otelOperator.mode=existing \ + --state-values-set metricsBackend.mode=existing \ + --state-values-set metricsBackend.type=external \ + --state-values-set-string metricsBackend.remoteWriteEndpoint=https://metrics.example.com/write \ + --state-values-set-string metricsBackend.promqlEndpoint=https://metrics.example.com \ + --state-values-set metricsBackend.authentication.mode=token \ + --state-values-set-string metricsBackend.authentication.authnEndpoint=https://auth.example.com \ + --selector name=observability-contract \ + template --output-dir "$work_dir/token-contract" >/dev/null + +token_contract="$(find "$work_dir/token-contract" -type f -name '*.yaml' -print)" +grep -q 'TIMESERIES_DB__AUTH_MODE: "token"' $token_contract || + fail "token auth mode was not published for the autoscaler" +grep -q 'TIMESERIES_DB__AUTHN_URL: "https://auth.example.com"' $token_contract || + fail "token auth endpoint was not published for the autoscaler" + +if HELMFILE_ENV=local helmfile \ + --file "$stack_dir/helmfile.d" \ + --environment default \ + --state-values-set observability.profile=invalid \ + list >"$work_dir/invalid-profile.log" 2>&1; then + fail "invalid profile was accepted" +fi +grep -q 'observability.profile must be disabled, control, compute, or all' \ + "$work_dir/invalid-profile.log" || + fail "invalid profile did not return the expected error" + +if HELMFILE_ENV=local helmfile \ + --file "$stack_dir/helmfile.d" \ + --environment default \ + --state-values-set observability.profile=disabled \ + --state-values-set observability.components.collector.mode=install \ + list >"$work_dir/disabled-override.log" 2>&1; then + fail "disabled profile accepted an install override" +fi +grep -q 'cannot be "install" when observability.profile=disabled' \ + "$work_dir/disabled-override.log" || + fail "disabled install override did not return the expected error" + +if HELMFILE_ENV=local helmfile \ + --file "$stack_dir/helmfile.d" \ + --environment default \ + --state-values-set observability.profile=control \ + --state-values-set metricsBackend.authentication.mode=invalid \ + list >"$work_dir/invalid-auth-mode.log" 2>&1; then + fail "invalid metrics backend auth mode was accepted" +fi +grep -q 'metricsBackend.authentication.mode must be none, token, or mtls' \ + "$work_dir/invalid-auth-mode.log" || + fail "invalid metrics backend auth mode did not return the expected error" + +echo "profile-defaults: all checks passed" diff --git a/deploy/stacks/self-managed/environments/base.yaml b/deploy/stacks/self-managed/environments/base.yaml index adcc0958b..924694ea6 100644 --- a/deploy/stacks/self-managed/environments/base.yaml +++ b/deploy/stacks/self-managed/environments/base.yaml @@ -133,29 +133,26 @@ global: # global.observability.metrics.enabled enabled: false -# Shared observability infrastructure. Self-managed control-plane installs keep -# this on because the function autoscaler depends on a populated TSDB. +# One profile installs and configures the shared observability stack for the +# selected planes. Self-managed control-plane installs need no other +# observability flags. observability: - mode: install + profile: control -prometheusOperatorCrds: - # Disable this when the cluster already owns Prometheus Operator CRDs, such as - # through an existing kube-prometheus-stack install, to avoid CRD conflicts. - enabled: true - -opentelemetryOperator: - enabled: true - -controlPlaneCollector: - enabled: true - -victoriaMetrics: - enabled: true - -defaultMonitors: - enabled: true - controlPlane: - enabled: true +# Defaults consumed only for the control and all observability profiles. +functionAutoscaler: + # The first chart release containing external envFrom support. The chart's + # appVersion pins the compatible autoscaler image. + chartVersion: "0.2.0" + region: local + cassandra: + contactPoints: cassandra.cassandra-system.svc.cluster.local + isDevelopment: false + nvcfApi: + grpcAddress: http://api.nvcf.svc.cluster.local:9090 + # Self-managed NVCF currently uses its internal unauthenticated gRPC path. + disableAuth: true + dryRun: false accounts: limits: diff --git a/deploy/stacks/self-managed/global.yaml.gotmpl b/deploy/stacks/self-managed/global.yaml.gotmpl index 19931401f..49e2c6f53 100644 --- a/deploy/stacks/self-managed/global.yaml.gotmpl +++ b/deploy/stacks/self-managed/global.yaml.gotmpl @@ -589,6 +589,44 @@ stateMetrics: serviceMonitor: enabled: false +{{- $observabilityContract := dict + "profile" (dig "observability" "profile" "disabled" .Values) + "metricsBackend" (dig "metricsBackend" dict .Values) + "autoscalerIntegration" (dig "autoscalerIntegration" dict .Values) }} +functionautoscaler: + namespace: nvcf + fullnameOverride: function-autoscaler + {{- if .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- toYaml .Values.global.imagePullSecrets | nindent 4 }} + {{- end }} + image: + registry: {{ .Values.global.image.registry }} + repository: {{ .Values.global.image.repository }}/nvcf-function-autoscaler + {{- with dig "functionAutoscaler" "image" "tag" nil .Values }} + tag: {{ . | quote }} + {{- end }} + podAnnotations: + checksum/observability-contract: {{ toJson $observabilityContract | sha256sum | quote }} + env: + CONFIG: /etc/server/config/settings-local.yaml + SECRETS_PATH: /vault/secrets/secrets.json + REGION: {{ dig "functionAutoscaler" "region" "local" .Values | quote }} + CASSANDRA__CONTACT_POINTS: {{ dig "functionAutoscaler" "cassandra" "contactPoints" "cassandra.cassandra-system.svc.cluster.local" .Values | quote }} + CASSANDRA__IS_DEVELOPMENT: {{ dig "functionAutoscaler" "cassandra" "isDevelopment" false .Values | quote }} + NVCF_API__NVCF_API_GRPC_ADDRESS: {{ dig "functionAutoscaler" "nvcfApi" "grpcAddress" "http://api.nvcf.svc.cluster.local:9090" .Values | quote }} + NVCF_API__DISABLE_AUTH: {{ dig "functionAutoscaler" "nvcfApi" "disableAuth" true .Values | quote }} + NVCF_API__DRY_RUN: {{ dig "functionAutoscaler" "nvcfApi" "dryRun" false .Values | quote }} + envFrom: + - configMapRef: + name: {{ dig "autoscalerIntegration" "configMapName" "nvcf-observability-autoscaler" .Values }} + {{- with include "nvcf.nodeSelector" (dict "type" "controlplane" "selectors" .Values.global.nodeSelectors) }} + {{- . | nindent 2 }} + {{- end }} + {{- with include "nvcf.tolerations" (dict "type" "controlplane" "tolerations" .Values.global.tolerations) }} + {{- . | nindent 2 }} + {{- end }} + reval: fullnameOverride: reval {{- if .Values.global.imagePullSecrets }} diff --git a/deploy/stacks/self-managed/helmfile.d/00-observability-infrastructure.yaml.gotmpl b/deploy/stacks/self-managed/helmfile.d/00-observability-infrastructure.yaml.gotmpl index 1fcd11c69..6f69b5ff4 100644 --- a/deploy/stacks/self-managed/helmfile.d/00-observability-infrastructure.yaml.gotmpl +++ b/deploy/stacks/self-managed/helmfile.d/00-observability-infrastructure.yaml.gotmpl @@ -7,11 +7,11 @@ environments: --- {{- $observability := dig "observability" (dict) .Values }} -{{- $mode := dig "mode" "disabled" $observability }} -{{- if not (has $mode (list "install" "existing" "disabled")) }} -{{- fail (printf "observability.mode must be install, existing, or disabled, got %q" $mode) }} +{{- $profile := dig "profile" "disabled" $observability }} +{{- if not (has $profile (list "disabled" "control" "compute" "all")) }} +{{- fail (printf "observability.profile must be disabled, control, compute, or all, got %q" $profile) }} {{- end }} -{{- if ne $mode "disabled" }} +{{- if ne $profile "disabled" }} helmfiles: - path: ../../observability/helmfile.d/01-observability.yaml.gotmpl selectorsInherited: true @@ -20,7 +20,7 @@ helmfiles: {{- toYaml (dig "global" (dict) .Values) | nindent 10 }} - observability: {{- toYaml $observability | nindent 10 }} - {{- range $key := list "prometheusOperatorCrds" "opentelemetryOperator" "controlPlaneCollector" "victoriaMetrics" "defaultMonitors" }} + {{- range $key := list "metricsBackend" "autoscalerIntegration" "prometheusOperatorCrds" "opentelemetryOperator" "collector" "victoriaMetrics" "defaultMonitors" }} {{- with (index $.Values $key) }} - {{ $key }}: {{- toYaml . | nindent 10 }} diff --git a/deploy/stacks/self-managed/helmfile.d/03-observability.yaml.gotmpl b/deploy/stacks/self-managed/helmfile.d/03-observability.yaml.gotmpl index 0d649fe9d..8ed7a655a 100644 --- a/deploy/stacks/self-managed/helmfile.d/03-observability.yaml.gotmpl +++ b/deploy/stacks/self-managed/helmfile.d/03-observability.yaml.gotmpl @@ -6,6 +6,12 @@ environments: --- +{{- $observabilityProfile := dig "observability" "profile" "disabled" .Values }} +{{- if not (has $observabilityProfile (list "disabled" "control" "compute" "all")) }} +{{- fail (printf "observability.profile must be disabled, control, compute, or all, got %q" $observabilityProfile) }} +{{- end }} +{{- $functionAutoscalerEnabled := or (eq $observabilityProfile "control") (eq $observabilityProfile "all") }} + repositories: - name: nvcf {{- if dig "global" "helm" "sources" "url" "" .Values }} @@ -38,6 +44,12 @@ templates: - ../secrets/{{ requiredEnv "HELMFILE_ENV" }}-secrets.yaml labels: release-group: observability + functionAutoscaler: &functionAutoscaler + chart: nvcf/helm-nvcf-function-autoscaler + values: + - ../global.yaml.gotmpl + labels: + release-group: observability releases: @@ -50,3 +62,11 @@ releases: - template: service # When enabled, this service will export realtime function information that can be # scraped by a Prometheus ServiceMonitor + + {{- if $functionAutoscalerEnabled }} + - name: function-autoscaler + version: {{ dig "functionAutoscaler" "chartVersion" "0.2.0" .Values | quote }} + namespace: nvcf + inherit: + - template: functionAutoscaler + {{- end }} diff --git a/deploy/stacks/self-managed/tests/observability-autoscaler.sh b/deploy/stacks/self-managed/tests/observability-autoscaler.sh new file mode 100755 index 000000000..2b37d6152 --- /dev/null +++ b/deploy/stacks/self-managed/tests/observability-autoscaler.sh @@ -0,0 +1,116 @@ +#!/usr/bin/env bash +set -euo pipefail + +stack_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +repo_dir="$(cd "$stack_dir/../../.." && pwd)" +work_dir="$(mktemp -d)" +trap 'rm -rf "$work_dir"' EXIT + +fail() { + echo "observability-autoscaler: $*" >&2 + exit 1 +} + +helmfile_args=( + --file "$stack_dir/helmfile.d" + --environment default + --allow-no-matching-release +) + +state_values=( + --state-values-set ingress.gatewayApi.gateways.shared.name=shared + --state-values-set ingress.gatewayApi.gateways.shared.namespace=envoy + --state-values-set ingress.gatewayApi.gateways.grpc.name=grpc + --state-values-set ingress.gatewayApi.gateways.grpc.namespace=envoy +) + +release_count() { + local profile="$1" + local release="$2" + + HELMFILE_ENV=base helmfile \ + "${helmfile_args[@]}" \ + "${state_values[@]}" \ + --state-values-set "observability.profile=$profile" \ + list 2>/dev/null | + awk -v release="$release" 'NR > 1 && $1 == release {count++} END {print count + 0}' +} + +release_namespace() { + local profile="$1" + local release="$2" + + HELMFILE_ENV=base helmfile \ + "${helmfile_args[@]}" \ + "${state_values[@]}" \ + --state-values-set "observability.profile=$profile" \ + list 2>/dev/null | + awk -v release="$release" 'NR > 1 && $1 == release {print $2}' +} + +for profile in control compute all; do + test "$(release_count "$profile" victoria-metrics)" = "1" || + fail "$profile profile did not install exactly one shared metrics backend" +done + +test "$(release_count disabled victoria-metrics)" = "0" || + fail "disabled profile installed the shared metrics backend" +test "$(release_namespace control observability-contract)" = "nvcf" || + fail "control profile did not place the contract in the autoscaler namespace" +test "$(release_count control function-autoscaler)" = "1" || + fail "control profile did not install exactly one function autoscaler" +test "$(release_count all function-autoscaler)" = "1" || + fail "all profile did not install exactly one function autoscaler" +test "$(release_count compute function-autoscaler)" = "0" || + fail "compute profile installed the control-plane function autoscaler" +test "$(release_count disabled function-autoscaler)" = "0" || + fail "disabled profile installed the function autoscaler" + +HELMFILE_ENV=base HELMFILE_CACHE_HOME="$work_dir/helmfile-cache" helmfile \ + --file "$stack_dir/helmfile.d/03-observability.yaml.gotmpl" \ + --environment default \ + "${state_values[@]}" \ + --state-values-set observability.profile=control \ + --selector name=function-autoscaler \ + write-values \ + --output-file-template "$work_dir/autoscaler-values.yaml" >/dev/null + +autoscaler_values="$work_dir/autoscaler-values.yaml" +for expected in \ + 'CASSANDRA__CONTACT_POINTS: cassandra.cassandra-system.svc.cluster.local' \ + 'CASSANDRA__IS_DEVELOPMENT: "false"' \ + 'NVCF_API__NVCF_API_GRPC_ADDRESS: http://api.nvcf.svc.cluster.local:9090' \ + 'NVCF_API__DISABLE_AUTH: "true"' \ + 'NVCF_API__DRY_RUN: "false"' \ + 'name: nvcf-observability-autoscaler'; do + grep -q "$expected" "$autoscaler_values" || + fail "control profile did not render autoscaler value: $expected" +done + +helm template function-autoscaler "$repo_dir/deploy/helm/function-autoscaler" \ + --namespace nvcf \ + --values "$autoscaler_values" \ + >"$work_dir/autoscaler-manifests.yaml" + +autoscaler_manifests="$work_dir/autoscaler-manifests.yaml" +grep -q 'image: nvcr.io/YOUR_ORG/YOUR_TEAM/nvcf-function-autoscaler:1.18.3' "$autoscaler_manifests" || + fail "autoscaler chart did not use its pinned appVersion" +test "$(grep -c 'name: nvcf-observability-autoscaler' "$autoscaler_manifests")" = "1" || + fail "autoscaler chart did not render exactly one external observability envFrom" +grep -q 'NVCF_API__DRY_RUN: "false"' "$autoscaler_manifests" || + fail "autoscaler chart did not render the self-managed runtime configuration" +grep -q '"helm.sh/hook": test' "$autoscaler_manifests" || + fail "autoscaler chart lost its runtime helm test hook" + +if HELMFILE_ENV=base helmfile \ + "${helmfile_args[@]}" \ + "${state_values[@]}" \ + --state-values-set observability.profile=invalid \ + list >"$work_dir/invalid-profile.log" 2>&1; then + fail "invalid profile was accepted by the self-managed stack" +fi +grep -q 'observability.profile must be disabled, control, compute, or all' \ + "$work_dir/invalid-profile.log" || + fail "invalid self-managed profile did not return the expected error" + +echo "observability-autoscaler: all checks passed" From b42d1135057a5adbc7f14285262123486afab223 Mon Sep 17 00:00:00 2001 From: Kristina Pathak Date: Tue, 28 Jul 2026 13:24:07 -0700 Subject: [PATCH 02/13] docs(skill): remove obsolete import routing Signed-off-by: Kristina Pathak --- .../dev/skills/nvcf-explore-stack/SKILL.md | 24 ++++++------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/ai-tooling/dev/skills/nvcf-explore-stack/SKILL.md b/ai-tooling/dev/skills/nvcf-explore-stack/SKILL.md index 4e5fa9668..098a1ce8f 100644 --- a/ai-tooling/dev/skills/nvcf-explore-stack/SKILL.md +++ b/ai-tooling/dev/skills/nvcf-explore-stack/SKILL.md @@ -6,7 +6,7 @@ description: >- namespaces, and `needs:` dependency chains. Reads deploy/stacks/self-managed/helmfile.d/*.yaml.gotmpl and deploy/stacks/nvcf-compute-plane/helmfile.d/*.yaml.gotmpl as the source of - truth for ordering and versions, with imports.yaml for upstream provenance. + truth for ordering and versions. Use when a user or developer asks "what deploys X", "what does X depend on", "what hooks run for X", "walk me through deployment order", "which subtree do I edit to change X", "what namespaces does the stack use", or @@ -39,8 +39,7 @@ Use this skill long enough to answer the question, then hand off to the right ex ## Required inputs -Read these from the monorepo root. Internal checkouts include `imports.yaml`; -the public snapshot may omit it. +Read these from the monorepo root. Authoritative (always read first when answering): @@ -52,20 +51,11 @@ Authoritative (always read first when answering): - `deploy/stacks/nvcf-compute-plane/helmfile.d/01-dependencies.yaml.gotmpl` - `deploy/stacks/nvcf-compute-plane/helmfile.d/02-nvca.yaml.gotmpl` -Provenance (when asked which subtree is monorepo-native vs. upstream-owned and -the file is present): - -- `imports.yaml` - Chart-level (when the chart is checked into the monorepo): - `deploy/helm//Chart.yaml` - `deploy/helm//values.yaml` -If workspace routing metadata is available and disagrees with the helmfile, the -helmfile wins. Update stale routing metadata in the same change rather than -guessing. - ## Common questions What deploys X @@ -75,16 +65,16 @@ What does X depend on : Return the `needs:` chain for that release plus the stage gate it sits behind (control-plane stages 0 -> 1 -> 2 -> 3, then compute-plane). Include any profile, `condition:`, or component mode that gates whether X deploys at all. What hooks run for X -: Read the checked-in chart under `deploy/helm//` when available. Search its `templates/` directory for Helm hook annotations, weights, hook events (pre-install / post-install), images used, and purpose. Cite the chart-relative template file path. If the chart is consumed from OCI only and the local workspace has routing metadata, use the workspace metadata and cite it. +: Read the checked-in chart under `deploy/helm//` when available. Search its `templates/` directory for Helm hook annotations, weights, hook events (pre-install / post-install), images used, and purpose. Cite the chart-relative template file path. If the chart is not checked in, cite its Helmfile chart reference and state that local templates are unavailable. Walk me through the full deployment order : Summarize control-plane stages 0 through 3 from the self-managed gotmpl files. Stage 0 delegates to the shared observability Helmfile when `observability.profile` is enabled. Stage 3 installs State Metrics and the function autoscaler for `control` and `all`. Then summarize the compute-plane stage from `deploy/stacks/nvcf-compute-plane/helmfile.d/01-dependencies.yaml.gotmpl` and `deploy/stacks/nvcf-compute-plane/helmfile.d/02-nvca.yaml.gotmpl`. Call out which releases run in parallel inside a stage and which are serialized by `needs:`. Which subtree do I edit to change X -: Two answers, both are important. For chart wiring (Helm hooks, manifests, values, hook weights) point at `deploy/helm//` if the chart is checked in, or note `oci-only` (the chart is consumed from the OCI registry and does not live in the monorepo). For runtime application logic, use the chart image repository, imports.yaml, and any available workspace routing metadata. `authoritative_source: native` means edits land here. `upstream` means edits also flow back to the upstream repo through the internal source-sync workflow. +: Point at the Helmfile path for orchestration, `deploy/helm//` for checked-in chart wiring, and the chart image repository under `src/`, `infra/`, or `migrations/` for runtime behavior. All stack sources are monorepo-native, so edits land in this repository. What namespaces does the stack use -: Return the list from the helmfile (`namespace:` per release). If workspace routing metadata includes a destroy namespace list, include it as supplemental context and cite that source. +: Return the list from the helmfile (`namespace:` per release). ## Subtree mapping @@ -93,8 +83,8 @@ The stack lives in three layers across the monorepo: | Concern | Lives at | |---------|----------| | Helmfile orchestration (stage ordering, env wiring, secrets flow) | `deploy/stacks/self-managed/` | -| Chart manifests, helm hooks, values | `deploy/helm//` (when vendored) or OCI registry only | -| Runtime application code, migrations | `src/`, `infra/`, `migrations/` (per `imports.yaml`) | +| Chart manifests, helm hooks, values | `deploy/helm//` when checked in; otherwise use the Helmfile chart reference | +| Runtime application code, migrations | `src/`, `infra/`, `migrations/` | When a question crosses layers, answer by layer and tell the user the order to edit (chart wiring first if the deploy contract changes, image source if behavior changes). From 9381159c8066db777aaace52438a75c5b1e34708 Mon Sep 17 00:00:00 2001 From: Kristina Pathak Date: Tue, 28 Jul 2026 13:28:58 -0700 Subject: [PATCH 03/13] fix(self-managed): keep autoscaler image pin stack-owned Signed-off-by: Kristina Pathak --- deploy/helm/function-autoscaler/Chart.yaml | 2 +- deploy/stacks/self-managed/environments/base.yaml | 6 ++++-- .../stacks/self-managed/tests/observability-autoscaler.sh | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/deploy/helm/function-autoscaler/Chart.yaml b/deploy/helm/function-autoscaler/Chart.yaml index 482ff68fe..0c824eb80 100644 --- a/deploy/helm/function-autoscaler/Chart.yaml +++ b/deploy/helm/function-autoscaler/Chart.yaml @@ -19,4 +19,4 @@ description: A Helm chart for NVCF Function Autoscaler Service type: application version: 0.0.0 # autoversioning enabled via release pipeline -appVersion: "1.18.3" +appVersion: "0.0.0" diff --git a/deploy/stacks/self-managed/environments/base.yaml b/deploy/stacks/self-managed/environments/base.yaml index 924694ea6..033a417a5 100644 --- a/deploy/stacks/self-managed/environments/base.yaml +++ b/deploy/stacks/self-managed/environments/base.yaml @@ -141,9 +141,11 @@ observability: # Defaults consumed only for the control and all observability profiles. functionAutoscaler: - # The first chart release containing external envFrom support. The chart's - # appVersion pins the compatible autoscaler image. + # The first chart release containing external envFrom support. chartVersion: "0.2.0" + image: + # Keep the stack's image pin independent of the chart's release metadata. + tag: "1.18.3" region: local cassandra: contactPoints: cassandra.cassandra-system.svc.cluster.local diff --git a/deploy/stacks/self-managed/tests/observability-autoscaler.sh b/deploy/stacks/self-managed/tests/observability-autoscaler.sh index 2b37d6152..38b599703 100755 --- a/deploy/stacks/self-managed/tests/observability-autoscaler.sh +++ b/deploy/stacks/self-managed/tests/observability-autoscaler.sh @@ -94,7 +94,7 @@ helm template function-autoscaler "$repo_dir/deploy/helm/function-autoscaler" \ autoscaler_manifests="$work_dir/autoscaler-manifests.yaml" grep -q 'image: nvcr.io/YOUR_ORG/YOUR_TEAM/nvcf-function-autoscaler:1.18.3' "$autoscaler_manifests" || - fail "autoscaler chart did not use its pinned appVersion" + fail "self-managed stack did not pin the autoscaler image" test "$(grep -c 'name: nvcf-observability-autoscaler' "$autoscaler_manifests")" = "1" || fail "autoscaler chart did not render exactly one external observability envFrom" grep -q 'NVCF_API__DRY_RUN: "false"' "$autoscaler_manifests" || From 666616328cc31eff34aa3150152599e794e9a9ac Mon Sep 17 00:00:00 2001 From: Kristina Pathak Date: Tue, 28 Jul 2026 13:36:39 -0700 Subject: [PATCH 04/13] fix(observability): enable selected plane monitors by default Signed-off-by: Kristina Pathak --- deploy/stacks/observability/README.md | 4 +++ .../charts/nvcf-default-monitors/values.yaml | 5 ++- .../observability/tests/profile-defaults.sh | 36 ++++++++++++------- 3 files changed, 30 insertions(+), 15 deletions(-) diff --git a/deploy/stacks/observability/README.md b/deploy/stacks/observability/README.md index cef84aa56..7a2aada9d 100644 --- a/deploy/stacks/observability/README.md +++ b/deploy/stacks/observability/README.md @@ -59,6 +59,10 @@ The profile-specific defaults are: | Autoscaler integration | Off | On | Off | On | | NVCA collector | Off | Off | Off | Off | +Selecting a plane automatically enables its complete default monitor set. No +separate plane-monitor flag is required. Individual monitors remain +fine-grained opt-outs, and `all` enables both monitor sets. + Control-plane monitoring covers State Metric Service, Invocation Service, gRPC Proxy, and LLM API Gateway. Compute monitoring selects NVCA in `nvca-system`, DCGM pods that carry NVCA's DCGM metrics label, and pods exposing a `metrics` diff --git a/deploy/stacks/observability/charts/nvcf-default-monitors/values.yaml b/deploy/stacks/observability/charts/nvcf-default-monitors/values.yaml index 81e287285..b8613b77a 100644 --- a/deploy/stacks/observability/charts/nvcf-default-monitors/values.yaml +++ b/deploy/stacks/observability/charts/nvcf-default-monitors/values.yaml @@ -17,14 +17,13 @@ commonLabels: nvcf.nvidia.com/observability-target: "true" dcgm: - enabled: false + enabled: true namespace: "" namespaceSelector: any: true interval: 30s path: /metrics - # NVCA currently uses this name for the injected DCGM metrics port; keep the - # monitor disabled until the consuming stack owns that chart contract. + # NVCA uses this name for the injected DCGM metrics port. port: dcgm-metrics labels: {} diff --git a/deploy/stacks/observability/tests/profile-defaults.sh b/deploy/stacks/observability/tests/profile-defaults.sh index 7a7a8dce7..646d77c6c 100755 --- a/deploy/stacks/observability/tests/profile-defaults.sh +++ b/deploy/stacks/observability/tests/profile-defaults.sh @@ -67,29 +67,41 @@ for profile in control compute all; do render_monitors "$profile" done +helm template default-monitors "$stack_dir/charts/nvcf-default-monitors" \ + --set controlPlane.enabled=true >"$work_dir/chart-control-defaults.yaml" +helm template default-monitors "$stack_dir/charts/nvcf-default-monitors" \ + --set computePlane.enabled=true >"$work_dir/chart-compute-defaults.yaml" + control_manifests="$(find "$work_dir/control" -type f -name '*.yaml' -print)" compute_manifests="$(find "$work_dir/compute" -type f -name '*.yaml' -print)" all_manifests="$(find "$work_dir/all" -type f -name '*.yaml' -print)" - -grep -q 'nvcf-default-monitors-state-metrics' $control_manifests || - fail "control profile is missing control-plane monitors" -if grep -q 'nvcf-default-monitors-nvca' $control_manifests; then - fail "control profile rendered compute-plane monitors" -fi +chart_control_manifests="$work_dir/chart-control-defaults.yaml" +chart_compute_manifests="$work_dir/chart-compute-defaults.yaml" + +for monitor in state-metrics invocation-service grpc-proxy llm-api-gateway; do + grep -q "nvcf-default-monitors-$monitor" "$chart_control_manifests" || + fail "monitor chart control defaults are missing $monitor monitor" + grep -q "nvcf-default-monitors-$monitor" $control_manifests || + fail "control profile is missing $monitor monitor" + grep -q "nvcf-default-monitors-$monitor" $all_manifests || + fail "all profile is missing $monitor monitor" + if grep -q "nvcf-default-monitors-$monitor" $compute_manifests; then + fail "compute profile rendered $monitor control-plane monitor" + fi +done for monitor in nvca dcgm worker; do + grep -q "nvcf-default-monitors-$monitor" "$chart_compute_manifests" || + fail "monitor chart compute defaults are missing $monitor monitor" grep -q "nvcf-default-monitors-$monitor" $compute_manifests || fail "compute profile is missing $monitor monitor" grep -q "nvcf-default-monitors-$monitor" $all_manifests || fail "all profile is missing $monitor monitor" + if grep -q "nvcf-default-monitors-$monitor" $control_manifests; then + fail "control profile rendered $monitor compute-plane monitor" + fi done -if grep -q 'nvcf-default-monitors-state-metrics' $compute_manifests; then - fail "compute profile rendered control-plane monitors" -fi -grep -q 'nvcf-default-monitors-state-metrics' $all_manifests || - fail "all profile is missing control-plane monitors" - for profile in control compute all; do HELMFILE_ENV=local helmfile \ --file "$stack_dir/helmfile.d" \ From 270297e837ddb97822b8e6cd57105e8cc89a04d2 Mon Sep 17 00:00:00 2001 From: Kristina Pathak Date: Tue, 28 Jul 2026 13:47:48 -0700 Subject: [PATCH 05/13] fix(observability): preserve explicit monitor overrides Signed-off-by: Kristina Pathak --- deploy/stacks/observability/README.md | 24 ++++++++++++-- .../charts/nvcf-default-monitors/values.yaml | 4 +-- .../helmfile.d/01-observability.yaml.gotmpl | 6 ++-- .../observability/tests/profile-defaults.sh | 32 +++++++++++++++++++ 4 files changed, 59 insertions(+), 7 deletions(-) diff --git a/deploy/stacks/observability/README.md b/deploy/stacks/observability/README.md index 7a2aada9d..4c5d4c905 100644 --- a/deploy/stacks/observability/README.md +++ b/deploy/stacks/observability/README.md @@ -59,9 +59,10 @@ The profile-specific defaults are: | Autoscaler integration | Off | On | Off | On | | NVCA collector | Off | Off | Off | Off | -Selecting a plane automatically enables its complete default monitor set. No -separate plane-monitor flag is required. Individual monitors remain -fine-grained opt-outs, and `all` enables both monitor sets. +Selecting a plane defaults its complete monitor set on. No separate +plane-monitor value is required for the normal path, and `all` defaults both +sets on. Explicit `defaultMonitors.controlPlane.enabled` and +`defaultMonitors.computePlane.enabled` values override the profile defaults. Control-plane monitoring covers State Metric Service, Invocation Service, gRPC Proxy, and LLM API Gateway. Compute monitoring selects NVCA in `nvca-system`, @@ -83,6 +84,23 @@ result without reimplementing profile logic or installing this stack again. Profiles provide normal defaults. Advanced installations can override shared component ownership with `install`, `existing`, or `disabled`. +Monitor groups can also be overridden without changing the selected profile: + +```yaml +observability: + profile: control + +defaultMonitors: + controlPlane: + enabled: false + computePlane: + enabled: true +``` + +These values control only monitor rendering. They do not change the selected +plane contract, BYOO support, or autoscaler integration. The `disabled` profile +still omits the entire observability release. + The fully expanded ownership configuration below is equivalent to setting only `observability.profile: control`: diff --git a/deploy/stacks/observability/charts/nvcf-default-monitors/values.yaml b/deploy/stacks/observability/charts/nvcf-default-monitors/values.yaml index b8613b77a..f1cc2170c 100644 --- a/deploy/stacks/observability/charts/nvcf-default-monitors/values.yaml +++ b/deploy/stacks/observability/charts/nvcf-default-monitors/values.yaml @@ -27,8 +27,8 @@ dcgm: port: dcgm-metrics labels: {} -# These plane gates are internal inputs derived from observability.profile by -# the parent stack. They are not public user-facing switches. +# The parent stack derives these monitor-group defaults from +# observability.profile. Explicit user values override those profile defaults. controlPlane: enabled: false interval: 30s diff --git a/deploy/stacks/observability/helmfile.d/01-observability.yaml.gotmpl b/deploy/stacks/observability/helmfile.d/01-observability.yaml.gotmpl index c8b712114..83f1520d8 100644 --- a/deploy/stacks/observability/helmfile.d/01-observability.yaml.gotmpl +++ b/deploy/stacks/observability/helmfile.d/01-observability.yaml.gotmpl @@ -114,10 +114,12 @@ environments: {{- $_ := set $collectorConfig "exporters" $collectorExporters }} {{- $defaultMonitors := deepCopy (dig "defaultMonitors" dict .Values) }} {{- $controlPlaneMonitors := deepCopy (dig "controlPlane" dict $defaultMonitors) }} -{{- $_ := set $controlPlaneMonitors "enabled" $controlEnabled }} +{{- $controlPlaneMonitorsEnabled := dig "defaultMonitors" "controlPlane" "enabled" $controlEnabled .Values }} +{{- $_ := set $controlPlaneMonitors "enabled" $controlPlaneMonitorsEnabled }} {{- $_ := set $defaultMonitors "controlPlane" $controlPlaneMonitors }} {{- $computePlaneMonitors := deepCopy (dig "computePlane" dict $defaultMonitors) }} -{{- $_ := set $computePlaneMonitors "enabled" $computeEnabled }} +{{- $computePlaneMonitorsEnabled := dig "defaultMonitors" "computePlane" "enabled" $computeEnabled .Values }} +{{- $_ := set $computePlaneMonitors "enabled" $computePlaneMonitorsEnabled }} {{- $_ := set $defaultMonitors "computePlane" $computePlaneMonitors }} repositories: diff --git a/deploy/stacks/observability/tests/profile-defaults.sh b/deploy/stacks/observability/tests/profile-defaults.sh index 646d77c6c..cf7692ce8 100755 --- a/deploy/stacks/observability/tests/profile-defaults.sh +++ b/deploy/stacks/observability/tests/profile-defaults.sh @@ -44,6 +44,23 @@ render_monitors() { template --output-dir "$output_dir" >/dev/null } +render_monitor_overrides() { + local profile="$1" + local output_name="$2" + local control_enabled="$3" + local compute_enabled="$4" + local output_dir="$work_dir/$output_name" + + HELMFILE_ENV=local helmfile \ + --file "$stack_dir/helmfile.d" \ + --environment default \ + --state-values-set "observability.profile=$profile" \ + --state-values-set "defaultMonitors.controlPlane.enabled=$control_enabled" \ + --state-values-set "defaultMonitors.computePlane.enabled=$compute_enabled" \ + --selector name=default-monitors \ + template --output-dir "$output_dir" >/dev/null +} + enabled_releases="$( cat <<'EOF' default-monitors @@ -67,6 +84,9 @@ for profile in control compute all; do render_monitors "$profile" done +render_monitor_overrides control control-swapped false true +render_monitor_overrides compute compute-swapped true false + helm template default-monitors "$stack_dir/charts/nvcf-default-monitors" \ --set controlPlane.enabled=true >"$work_dir/chart-control-defaults.yaml" helm template default-monitors "$stack_dir/charts/nvcf-default-monitors" \ @@ -75,6 +95,8 @@ helm template default-monitors "$stack_dir/charts/nvcf-default-monitors" \ control_manifests="$(find "$work_dir/control" -type f -name '*.yaml' -print)" compute_manifests="$(find "$work_dir/compute" -type f -name '*.yaml' -print)" all_manifests="$(find "$work_dir/all" -type f -name '*.yaml' -print)" +control_swapped_manifests="$(find "$work_dir/control-swapped" -type f -name '*.yaml' -print)" +compute_swapped_manifests="$(find "$work_dir/compute-swapped" -type f -name '*.yaml' -print)" chart_control_manifests="$work_dir/chart-control-defaults.yaml" chart_compute_manifests="$work_dir/chart-compute-defaults.yaml" @@ -85,6 +107,11 @@ for monitor in state-metrics invocation-service grpc-proxy llm-api-gateway; do fail "control profile is missing $monitor monitor" grep -q "nvcf-default-monitors-$monitor" $all_manifests || fail "all profile is missing $monitor monitor" + grep -q "nvcf-default-monitors-$monitor" $compute_swapped_manifests || + fail "explicit control monitor override did not win for compute profile" + if grep -q "nvcf-default-monitors-$monitor" $control_swapped_manifests; then + fail "explicit control monitor override did not disable $monitor" + fi if grep -q "nvcf-default-monitors-$monitor" $compute_manifests; then fail "compute profile rendered $monitor control-plane monitor" fi @@ -97,6 +124,11 @@ for monitor in nvca dcgm worker; do fail "compute profile is missing $monitor monitor" grep -q "nvcf-default-monitors-$monitor" $all_manifests || fail "all profile is missing $monitor monitor" + grep -q "nvcf-default-monitors-$monitor" $control_swapped_manifests || + fail "explicit compute monitor override did not win for control profile" + if grep -q "nvcf-default-monitors-$monitor" $compute_swapped_manifests; then + fail "explicit compute monitor override did not disable $monitor" + fi if grep -q "nvcf-default-monitors-$monitor" $control_manifests; then fail "control profile rendered $monitor compute-plane monitor" fi From fd46c369ce0e52e404d0a777d9b98bf8da34c5c1 Mon Sep 17 00:00:00 2001 From: Kristina Pathak Date: Tue, 28 Jul 2026 13:53:46 -0700 Subject: [PATCH 06/13] refactor(observability): nest compute monitor values Signed-off-by: Kristina Pathak --- deploy/stacks/observability/README.md | 7 +- .../templates/dcgm-podmonitor.yaml | 14 ++-- .../templates/nvca-servicemonitor.yaml | 14 ++-- .../templates/worker-podmonitor.yaml | 14 ++-- .../charts/nvcf-default-monitors/values.yaml | 66 +++++++++---------- .../observability/environments/base.yaml | 61 ++++++++--------- .../observability/tests/profile-defaults.sh | 22 +++++++ 7 files changed, 113 insertions(+), 85 deletions(-) diff --git a/deploy/stacks/observability/README.md b/deploy/stacks/observability/README.md index 4c5d4c905..982a3d668 100644 --- a/deploy/stacks/observability/README.md +++ b/deploy/stacks/observability/README.md @@ -95,11 +95,16 @@ defaultMonitors: enabled: false computePlane: enabled: true + worker: + enabled: false ``` These values control only monitor rendering. They do not change the selected plane contract, BYOO support, or autoscaler integration. The `disabled` profile -still omits the entire observability release. +still omits the entire observability release. Compute monitor overrides live +under `defaultMonitors.computePlane.nvca`, +`defaultMonitors.computePlane.dcgm`, and +`defaultMonitors.computePlane.worker`. The fully expanded ownership configuration below is equivalent to setting only `observability.profile: control`: diff --git a/deploy/stacks/observability/charts/nvcf-default-monitors/templates/dcgm-podmonitor.yaml b/deploy/stacks/observability/charts/nvcf-default-monitors/templates/dcgm-podmonitor.yaml index 563c4618f..39f052b9e 100644 --- a/deploy/stacks/observability/charts/nvcf-default-monitors/templates/dcgm-podmonitor.yaml +++ b/deploy/stacks/observability/charts/nvcf-default-monitors/templates/dcgm-podmonitor.yaml @@ -15,24 +15,24 @@ See the License for the specific language governing permissions and limitations under the License. */}} -{{- if and .Values.computePlane.enabled .Values.dcgm.enabled }} +{{- if and .Values.computePlane.enabled .Values.computePlane.dcgm.enabled }} apiVersion: monitoring.coreos.com/v1 kind: PodMonitor metadata: name: {{ include "nvcf-default-monitors.name" . }}-dcgm - namespace: {{ default .Release.Namespace .Values.dcgm.namespace }} + namespace: {{ default .Release.Namespace .Values.computePlane.dcgm.namespace }} labels: {{- include "nvcf-default-monitors.labels" . | nindent 4 }} - {{- with .Values.dcgm.labels }} + {{- with .Values.computePlane.dcgm.labels }} {{- toYaml . | nindent 4 }} {{- end }} spec: podMetricsEndpoints: - - interval: {{ .Values.dcgm.interval | quote }} - path: {{ .Values.dcgm.path | quote }} - port: {{ .Values.dcgm.port | quote }} + - interval: {{ .Values.computePlane.dcgm.interval | quote }} + path: {{ .Values.computePlane.dcgm.path | quote }} + port: {{ .Values.computePlane.dcgm.port | quote }} namespaceSelector: - {{- toYaml .Values.dcgm.namespaceSelector | nindent 4 }} + {{- toYaml .Values.computePlane.dcgm.namespaceSelector | nindent 4 }} selector: matchLabels: nvca.nvcf.nvidia.io/dcgm-metrics-present: "true" diff --git a/deploy/stacks/observability/charts/nvcf-default-monitors/templates/nvca-servicemonitor.yaml b/deploy/stacks/observability/charts/nvcf-default-monitors/templates/nvca-servicemonitor.yaml index d51138b26..abd47c5f2 100644 --- a/deploy/stacks/observability/charts/nvcf-default-monitors/templates/nvca-servicemonitor.yaml +++ b/deploy/stacks/observability/charts/nvcf-default-monitors/templates/nvca-servicemonitor.yaml @@ -15,7 +15,7 @@ See the License for the specific language governing permissions and limitations under the License. */}} -{{- if and .Values.computePlane.enabled .Values.nvca.enabled }} +{{- if and .Values.computePlane.enabled .Values.computePlane.nvca.enabled }} apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor metadata: @@ -23,17 +23,17 @@ metadata: namespace: {{ .Release.Namespace }} labels: {{- include "nvcf-default-monitors.labels" . | nindent 4 }} - {{- with .Values.nvca.labels }} + {{- with .Values.computePlane.nvca.labels }} {{- toYaml . | nindent 4 }} {{- end }} spec: namespaceSelector: - {{- toYaml .Values.nvca.namespaceSelector | nindent 4 }} + {{- toYaml .Values.computePlane.nvca.namespaceSelector | nindent 4 }} selector: matchLabels: - {{- toYaml .Values.nvca.selector | nindent 6 }} + {{- toYaml .Values.computePlane.nvca.selector | nindent 6 }} endpoints: - - interval: {{ .Values.nvca.interval | quote }} - path: {{ .Values.nvca.path | quote }} - port: {{ .Values.nvca.port | quote }} + - interval: {{ .Values.computePlane.nvca.interval | quote }} + path: {{ .Values.computePlane.nvca.path | quote }} + port: {{ .Values.computePlane.nvca.port | quote }} {{- end }} diff --git a/deploy/stacks/observability/charts/nvcf-default-monitors/templates/worker-podmonitor.yaml b/deploy/stacks/observability/charts/nvcf-default-monitors/templates/worker-podmonitor.yaml index 557feeddb..3f168a2d0 100644 --- a/deploy/stacks/observability/charts/nvcf-default-monitors/templates/worker-podmonitor.yaml +++ b/deploy/stacks/observability/charts/nvcf-default-monitors/templates/worker-podmonitor.yaml @@ -15,7 +15,7 @@ See the License for the specific language governing permissions and limitations under the License. */}} -{{- if and .Values.computePlane.enabled .Values.worker.enabled }} +{{- if and .Values.computePlane.enabled .Values.computePlane.worker.enabled }} apiVersion: monitoring.coreos.com/v1 kind: PodMonitor metadata: @@ -23,16 +23,16 @@ metadata: namespace: {{ .Release.Namespace }} labels: {{- include "nvcf-default-monitors.labels" . | nindent 4 }} - {{- with .Values.worker.labels }} + {{- with .Values.computePlane.worker.labels }} {{- toYaml . | nindent 4 }} {{- end }} spec: namespaceSelector: - {{- toYaml .Values.worker.namespaceSelector | nindent 4 }} + {{- toYaml .Values.computePlane.worker.namespaceSelector | nindent 4 }} selector: - matchLabels: {{ .Values.worker.selector | toJson }} + matchLabels: {{ .Values.computePlane.worker.selector | toJson }} podMetricsEndpoints: - - interval: {{ .Values.worker.interval | quote }} - path: {{ .Values.worker.path | quote }} - port: {{ .Values.worker.port | quote }} + - interval: {{ .Values.computePlane.worker.interval | quote }} + path: {{ .Values.computePlane.worker.path | quote }} + port: {{ .Values.computePlane.worker.port | quote }} {{- end }} diff --git a/deploy/stacks/observability/charts/nvcf-default-monitors/values.yaml b/deploy/stacks/observability/charts/nvcf-default-monitors/values.yaml index f1cc2170c..e9542c2f4 100644 --- a/deploy/stacks/observability/charts/nvcf-default-monitors/values.yaml +++ b/deploy/stacks/observability/charts/nvcf-default-monitors/values.yaml @@ -16,17 +16,6 @@ commonLabels: nvcf.nvidia.com/observability-target: "true" -dcgm: - enabled: true - namespace: "" - namespaceSelector: - any: true - interval: 30s - path: /metrics - # NVCA uses this name for the injected DCGM metrics port. - port: dcgm-metrics - labels: {} - # The parent stack derives these monitor-group defaults from # observability.profile. Explicit user values override those profile defaults. controlPlane: @@ -60,26 +49,37 @@ controlPlane: computePlane: enabled: false -nvca: - enabled: true - namespaceSelector: - matchNames: - - nvca-system - interval: 30s - path: /metrics - port: nvca - selector: - app.kubernetes.io/name: nvca - labels: {} + nvca: + enabled: true + namespaceSelector: + matchNames: + - nvca-system + interval: 30s + path: /metrics + port: nvca + selector: + app.kubernetes.io/name: nvca + labels: {} -worker: - enabled: true - namespaceSelector: - matchExpressions: - - key: nvca.nvcf.nvidia.io/workload-instance-type - operator: Exists - interval: 30s - path: /metrics - port: metrics - selector: {} - labels: {} + dcgm: + enabled: true + namespace: "" + namespaceSelector: + any: true + interval: 30s + path: /metrics + # NVCA uses this name for the injected DCGM metrics port. + port: dcgm-metrics + labels: {} + + worker: + enabled: true + namespaceSelector: + matchExpressions: + - key: nvca.nvcf.nvidia.io/workload-instance-type + operator: Exists + interval: 30s + path: /metrics + port: metrics + selector: {} + labels: {} diff --git a/deploy/stacks/observability/environments/base.yaml b/deploy/stacks/observability/environments/base.yaml index b30aeeab5..18e992511 100644 --- a/deploy/stacks/observability/environments/base.yaml +++ b/deploy/stacks/observability/environments/base.yaml @@ -152,33 +152,34 @@ defaultMonitors: enabled: true selector: app.kubernetes.io/instance: llm-api-gateway - nvca: - enabled: true - namespaceSelector: - matchNames: - - nvca-system - interval: 30s - path: /metrics - port: nvca - selector: - app.kubernetes.io/name: nvca - labels: {} - dcgm: - enabled: true - namespaceSelector: - any: true - interval: 30s - path: /metrics - port: dcgm-metrics - labels: {} - worker: - enabled: true - namespaceSelector: - matchExpressions: - - key: nvca.nvcf.nvidia.io/workload-instance-type - operator: Exists - interval: 30s - path: /metrics - port: metrics - selector: {} - labels: {} + computePlane: + nvca: + enabled: true + namespaceSelector: + matchNames: + - nvca-system + interval: 30s + path: /metrics + port: nvca + selector: + app.kubernetes.io/name: nvca + labels: {} + dcgm: + enabled: true + namespaceSelector: + any: true + interval: 30s + path: /metrics + port: dcgm-metrics + labels: {} + worker: + enabled: true + namespaceSelector: + matchExpressions: + - key: nvca.nvcf.nvidia.io/workload-instance-type + operator: Exists + interval: 30s + path: /metrics + port: metrics + selector: {} + labels: {} diff --git a/deploy/stacks/observability/tests/profile-defaults.sh b/deploy/stacks/observability/tests/profile-defaults.sh index cf7692ce8..4f6c49172 100755 --- a/deploy/stacks/observability/tests/profile-defaults.sh +++ b/deploy/stacks/observability/tests/profile-defaults.sh @@ -61,6 +61,18 @@ render_monitor_overrides() { template --output-dir "$output_dir" >/dev/null } +render_compute_monitor_override() { + local output_dir="$work_dir/compute-worker-disabled" + + HELMFILE_ENV=local helmfile \ + --file "$stack_dir/helmfile.d" \ + --environment default \ + --state-values-set observability.profile=compute \ + --state-values-set defaultMonitors.computePlane.worker.enabled=false \ + --selector name=default-monitors \ + template --output-dir "$output_dir" >/dev/null +} + enabled_releases="$( cat <<'EOF' default-monitors @@ -86,6 +98,7 @@ done render_monitor_overrides control control-swapped false true render_monitor_overrides compute compute-swapped true false +render_compute_monitor_override helm template default-monitors "$stack_dir/charts/nvcf-default-monitors" \ --set controlPlane.enabled=true >"$work_dir/chart-control-defaults.yaml" @@ -97,6 +110,7 @@ compute_manifests="$(find "$work_dir/compute" -type f -name '*.yaml' -print)" all_manifests="$(find "$work_dir/all" -type f -name '*.yaml' -print)" control_swapped_manifests="$(find "$work_dir/control-swapped" -type f -name '*.yaml' -print)" compute_swapped_manifests="$(find "$work_dir/compute-swapped" -type f -name '*.yaml' -print)" +compute_worker_disabled_manifests="$(find "$work_dir/compute-worker-disabled" -type f -name '*.yaml' -print)" chart_control_manifests="$work_dir/chart-control-defaults.yaml" chart_compute_manifests="$work_dir/chart-compute-defaults.yaml" @@ -134,6 +148,14 @@ for monitor in nvca dcgm worker; do fi done +for monitor in nvca dcgm; do + grep -q "nvcf-default-monitors-$monitor" $compute_worker_disabled_manifests || + fail "nested worker override disabled the $monitor monitor" +done +if grep -q "nvcf-default-monitors-worker" $compute_worker_disabled_manifests; then + fail "nested worker monitor override was ignored" +fi + for profile in control compute all; do HELMFILE_ENV=local helmfile \ --file "$stack_dir/helmfile.d" \ From 1b52225b0f7f44a8e6a70bf166a8617359f0c11f Mon Sep 17 00:00:00 2001 From: Kristina Pathak Date: Tue, 28 Jul 2026 13:57:38 -0700 Subject: [PATCH 07/13] refactor(observability): share service monitor template Signed-off-by: Kristina Pathak --- deploy/stacks/observability/README.md | 2 +- .../templates/nvca-servicemonitor.yaml | 39 ------------------- ...vicemonitors.yaml => servicemonitors.yaml} | 22 +++++++---- .../charts/nvcf-default-monitors/values.yaml | 21 +++++----- .../observability/environments/base.yaml | 20 +++++----- .../observability/tests/profile-defaults.sh | 9 +++++ 6 files changed, 44 insertions(+), 69 deletions(-) delete mode 100644 deploy/stacks/observability/charts/nvcf-default-monitors/templates/nvca-servicemonitor.yaml rename deploy/stacks/observability/charts/nvcf-default-monitors/templates/{controlplane-servicemonitors.yaml => servicemonitors.yaml} (58%) diff --git a/deploy/stacks/observability/README.md b/deploy/stacks/observability/README.md index 982a3d668..fb7797bd4 100644 --- a/deploy/stacks/observability/README.md +++ b/deploy/stacks/observability/README.md @@ -102,7 +102,7 @@ defaultMonitors: These values control only monitor rendering. They do not change the selected plane contract, BYOO support, or autoscaler integration. The `disabled` profile still omits the entire observability release. Compute monitor overrides live -under `defaultMonitors.computePlane.nvca`, +under `defaultMonitors.computePlane.services` for ServiceMonitors, `defaultMonitors.computePlane.dcgm`, and `defaultMonitors.computePlane.worker`. diff --git a/deploy/stacks/observability/charts/nvcf-default-monitors/templates/nvca-servicemonitor.yaml b/deploy/stacks/observability/charts/nvcf-default-monitors/templates/nvca-servicemonitor.yaml deleted file mode 100644 index abd47c5f2..000000000 --- a/deploy/stacks/observability/charts/nvcf-default-monitors/templates/nvca-servicemonitor.yaml +++ /dev/null @@ -1,39 +0,0 @@ -{{/* -SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -SPDX-License-Identifier: Apache-2.0 - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - https://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/}} - -{{- if and .Values.computePlane.enabled .Values.computePlane.nvca.enabled }} -apiVersion: monitoring.coreos.com/v1 -kind: ServiceMonitor -metadata: - name: {{ include "nvcf-default-monitors.name" . }}-nvca - namespace: {{ .Release.Namespace }} - labels: - {{- include "nvcf-default-monitors.labels" . | nindent 4 }} - {{- with .Values.computePlane.nvca.labels }} - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - namespaceSelector: - {{- toYaml .Values.computePlane.nvca.namespaceSelector | nindent 4 }} - selector: - matchLabels: - {{- toYaml .Values.computePlane.nvca.selector | nindent 6 }} - endpoints: - - interval: {{ .Values.computePlane.nvca.interval | quote }} - path: {{ .Values.computePlane.nvca.path | quote }} - port: {{ .Values.computePlane.nvca.port | quote }} -{{- end }} diff --git a/deploy/stacks/observability/charts/nvcf-default-monitors/templates/controlplane-servicemonitors.yaml b/deploy/stacks/observability/charts/nvcf-default-monitors/templates/servicemonitors.yaml similarity index 58% rename from deploy/stacks/observability/charts/nvcf-default-monitors/templates/controlplane-servicemonitors.yaml rename to deploy/stacks/observability/charts/nvcf-default-monitors/templates/servicemonitors.yaml index c207cf424..4161ddcfd 100644 --- a/deploy/stacks/observability/charts/nvcf-default-monitors/templates/controlplane-servicemonitors.yaml +++ b/deploy/stacks/observability/charts/nvcf-default-monitors/templates/servicemonitors.yaml @@ -15,10 +15,15 @@ See the License for the specific language governing permissions and limitations under the License. */}} -{{- if .Values.controlPlane.enabled }} {{- $top := . }} -{{- $cp := .Values.controlPlane }} -{{- range $svc := $cp.services }} +{{- $groups := list + (dict "name" "controlPlane" "values" .Values.controlPlane) + (dict "name" "computePlane" "values" .Values.computePlane) }} +{{- range $group := $groups }} +{{- $groupName := get $group "name" }} +{{- $groupValues := get $group "values" }} +{{- if $groupValues.enabled }} +{{- range $svc := $groupValues.services }} {{- if $svc.enabled }} --- apiVersion: monitoring.coreos.com/v1 @@ -34,14 +39,15 @@ metadata: spec: namespaceSelector: matchNames: - {{- toYaml (default $cp.namespaces $svc.namespaces) | nindent 6 }} + {{- toYaml (required (printf "%s service %q must set namespaces" $groupName $svc.name) (default $groupValues.namespaces $svc.namespaces)) | nindent 6 }} selector: matchLabels: - {{- toYaml (required (printf "controlPlane service %q must set a selector" $svc.name) $svc.selector) | nindent 6 }} + {{- toYaml (required (printf "%s service %q must set a selector" $groupName $svc.name) $svc.selector) | nindent 6 }} endpoints: - - port: {{ $svc.port | default $cp.port | quote }} - path: {{ $svc.path | default $cp.path | quote }} - interval: {{ $svc.interval | default $cp.interval | quote }} + - port: {{ $svc.port | default $groupValues.port | quote }} + path: {{ $svc.path | default $groupValues.path | quote }} + interval: {{ $svc.interval | default $groupValues.interval | quote }} +{{- end }} {{- end }} {{- end }} {{- end }} diff --git a/deploy/stacks/observability/charts/nvcf-default-monitors/values.yaml b/deploy/stacks/observability/charts/nvcf-default-monitors/values.yaml index e9542c2f4..0bbdfd4c2 100644 --- a/deploy/stacks/observability/charts/nvcf-default-monitors/values.yaml +++ b/deploy/stacks/observability/charts/nvcf-default-monitors/values.yaml @@ -48,18 +48,17 @@ controlPlane: computePlane: enabled: false - - nvca: - enabled: true - namespaceSelector: - matchNames: + interval: 30s + path: /metrics + services: + - name: nvca + enabled: true + namespaces: - nvca-system - interval: 30s - path: /metrics - port: nvca - selector: - app.kubernetes.io/name: nvca - labels: {} + port: nvca + selector: + app.kubernetes.io/name: nvca + labels: {} dcgm: enabled: true diff --git a/deploy/stacks/observability/environments/base.yaml b/deploy/stacks/observability/environments/base.yaml index 18e992511..6faf8d49e 100644 --- a/deploy/stacks/observability/environments/base.yaml +++ b/deploy/stacks/observability/environments/base.yaml @@ -153,17 +153,17 @@ defaultMonitors: selector: app.kubernetes.io/instance: llm-api-gateway computePlane: - nvca: - enabled: true - namespaceSelector: - matchNames: + interval: 30s + path: /metrics + services: + - name: nvca + enabled: true + namespaces: - nvca-system - interval: 30s - path: /metrics - port: nvca - selector: - app.kubernetes.io/name: nvca - labels: {} + port: nvca + selector: + app.kubernetes.io/name: nvca + labels: {} dcgm: enabled: true namespaceSelector: diff --git a/deploy/stacks/observability/tests/profile-defaults.sh b/deploy/stacks/observability/tests/profile-defaults.sh index 4f6c49172..7c4a0c968 100755 --- a/deploy/stacks/observability/tests/profile-defaults.sh +++ b/deploy/stacks/observability/tests/profile-defaults.sh @@ -73,6 +73,15 @@ render_compute_monitor_override() { template --output-dir "$output_dir" >/dev/null } +service_monitor_template_count="$( + find "$stack_dir/charts/nvcf-default-monitors/templates" \ + -maxdepth 1 -type f -name '*servicemonitor*.yaml' | + wc -l | + tr -d ' ' +)" +test "$service_monitor_template_count" = "1" || + fail "default monitors chart must use one generic ServiceMonitor template" + enabled_releases="$( cat <<'EOF' default-monitors From 06842ea9c5f199a92abe80b69b86913cd370e62b Mon Sep 17 00:00:00 2001 From: Kristina Pathak Date: Tue, 28 Jul 2026 14:09:12 -0700 Subject: [PATCH 08/13] refactor(observability): consolidate monitor and autoscaler config --- .../templates/deployment.yaml | 7 +- deploy/helm/function-autoscaler/values.yaml | 7 -- deploy/stacks/observability/AGENTS.md | 1 - deploy/stacks/observability/Makefile | 1 - deploy/stacks/observability/README.md | 35 +++--- ...{dcgm-podmonitor.yaml => podmonitors.yaml} | 33 ++++-- .../templates/worker-podmonitor.yaml | 38 ------- .../charts/nvcf-default-monitors/values.yaml | 3 + .../nvcf-observability-contract/Chart.yaml | 21 ---- .../templates/autoscaler-configmap.yaml | 42 ------- .../templates/profile-configmap.yaml | 36 ------ .../nvcf-observability-contract/values.yaml | 32 ------ .../observability/environments/base.yaml | 10 +- .../helmfile.d/01-observability.yaml.gotmpl | 30 ----- .../observability/tests/profile-defaults.sh | 105 ++---------------- .../self-managed/environments/base.yaml | 7 +- deploy/stacks/self-managed/global.yaml.gotmpl | 30 +++-- ...0-observability-infrastructure.yaml.gotmpl | 2 +- .../helmfile.d/03-observability.yaml.gotmpl | 2 +- .../tests/observability-autoscaler.sh | 72 +++++++----- 20 files changed, 131 insertions(+), 383 deletions(-) rename deploy/stacks/observability/charts/nvcf-default-monitors/templates/{dcgm-podmonitor.yaml => podmonitors.yaml} (51%) delete mode 100644 deploy/stacks/observability/charts/nvcf-default-monitors/templates/worker-podmonitor.yaml delete mode 100644 deploy/stacks/observability/charts/nvcf-observability-contract/Chart.yaml delete mode 100644 deploy/stacks/observability/charts/nvcf-observability-contract/templates/autoscaler-configmap.yaml delete mode 100644 deploy/stacks/observability/charts/nvcf-observability-contract/templates/profile-configmap.yaml delete mode 100644 deploy/stacks/observability/charts/nvcf-observability-contract/values.yaml diff --git a/deploy/helm/function-autoscaler/templates/deployment.yaml b/deploy/helm/function-autoscaler/templates/deployment.yaml index 2b0fb675e..5f55f50de 100644 --- a/deploy/helm/function-autoscaler/templates/deployment.yaml +++ b/deploy/helm/function-autoscaler/templates/deployment.yaml @@ -66,16 +66,11 @@ spec: {{- toYaml .Values.functionautoscaler.readinessProbe | nindent 10 }} resources: {{- toYaml .Values.functionautoscaler.resources | nindent 10 }} - {{- if or .Values.functionautoscaler.env .Values.functionautoscaler.envFrom }} - envFrom: {{- if .Values.functionautoscaler.env }} + envFrom: - configMapRef: name: {{ include "function-autoscaler.fullname" . }}-env {{- end }} - {{- with .Values.functionautoscaler.envFrom }} - {{- toYaml . | nindent 8 }} - {{- end }} - {{- end }} {{- if or .Values.functionautoscaler.volumeMounts .Values.functionautoscaler.vault.enabled }} volumeMounts: {{- with .Values.functionautoscaler.volumeMounts }} diff --git a/deploy/helm/function-autoscaler/values.yaml b/deploy/helm/function-autoscaler/values.yaml index dd5e82767..db8eec8f7 100644 --- a/deploy/helm/function-autoscaler/values.yaml +++ b/deploy/helm/function-autoscaler/values.yaml @@ -140,10 +140,3 @@ functionautoscaler: # Read the secrets file rendered by the Vault Agent. Override this value # when mounting secrets manually. SECRETS_PATH: "/vault/secrets/secrets.json" - - # Additional ConfigMaps or Secrets to load after the chart-owned env - # ConfigMap. This lets a stack provide resolved infrastructure contracts - # without duplicating them in this chart's values. - envFrom: [] - # - configMapRef: - # name: nvcf-observability-autoscaler diff --git a/deploy/stacks/observability/AGENTS.md b/deploy/stacks/observability/AGENTS.md index e88dafa56..94184d524 100644 --- a/deploy/stacks/observability/AGENTS.md +++ b/deploy/stacks/observability/AGENTS.md @@ -28,7 +28,6 @@ deployments. - `environments/base.yaml`: scaffold defaults and scrape contracts - `charts/nvcf-otel-collector`: OpenTelemetryCollector resource and Target Allocator RBAC - `charts/nvcf-default-monitors`: centrally owned concrete monitor resources -- `charts/nvcf-observability-contract`: resolved settings for consumers such as the autoscaler - `values/victoria-metrics.yaml.gotmpl`: values bridge for the VictoriaMetrics chart ## Profiles diff --git a/deploy/stacks/observability/Makefile b/deploy/stacks/observability/Makefile index 706c25ec5..d910aa875 100644 --- a/deploy/stacks/observability/Makefile +++ b/deploy/stacks/observability/Makefile @@ -24,7 +24,6 @@ template: clean test: @helm lint charts/nvcf-default-monitors - @helm lint charts/nvcf-observability-contract @helm lint charts/nvcf-otel-collector @helm lint ../../helm/function-autoscaler \ --set functionautoscaler.image.registry=nvcr.io \ diff --git a/deploy/stacks/observability/README.md b/deploy/stacks/observability/README.md index fb7797bd4..5c486f486 100644 --- a/deploy/stacks/observability/README.md +++ b/deploy/stacks/observability/README.md @@ -22,7 +22,7 @@ The stack can own: | Profile | Control targets | Compute targets | Result | | --- | --- | --- | --- | | `disabled` | No | No | Render no observability releases or resources. | -| `control` | Yes | No | Install one stack for control-plane metrics and the autoscaler backend contract. | +| `control` | Yes | No | Install one stack for control-plane metrics and the autoscaler backend. | | `compute` | No | Yes | Install one stack for NVCA, DCGM, and worker metrics; resolve NVCA BYOO support on. | | `all` | Yes | Yes | Install the union once; do not duplicate shared components. | @@ -69,16 +69,11 @@ Proxy, and LLM API Gateway. Compute monitoring selects NVCA in `nvca-system`, DCGM pods that carry NVCA's DCGM metrics label, and pods exposing a `metrics` port in NVCA-managed workload namespaces. -For `compute` and `all`, the resolved profile contract defaults BYOO support on. +For `compute` and `all`, the profile defaults BYOO support on. The NVCA installer should map that value to its existing `BYOObservability` feature gate. The feature gate enables the per-function BYOO collector path; it does not create a second shared collector. This stack does not deploy NVCA. -Every enabled profile publishes its resolved consumer defaults in the -`nvcf-observability-profile` ConfigMap in the contract namespace, `nvcf` by -default. That contract lets separately packaged consumers use the profile -result without reimplementing profile logic or installing this stack again. - ## Fine-Grained Overrides Profiles provide normal defaults. Advanced installations can override shared @@ -100,7 +95,7 @@ defaultMonitors: ``` These values control only monitor rendering. They do not change the selected -plane contract, BYOO support, or autoscaler integration. The `disabled` profile +profile, BYOO support, or autoscaler integration. The `disabled` profile still omits the entire observability release. Compute monitor overrides live under `defaultMonitors.computePlane.services` for ServiceMonitors, `defaultMonitors.computePlane.dcgm`, and @@ -170,12 +165,13 @@ PromQL: http://vmsingle.monitoring.svc.cluster.local:8428 An external backend must provide both endpoints for `control` and `all`. -## Autoscaler Contract +## Autoscaler Integration The autoscaler is a control-plane consumer, not a second observability stack. It reads from the same `metricsBackend.promqlEndpoint` selected by the profile. -For `control` and `all`, the stack publishes that resolved contract as -`nvcf-observability-autoscaler` in the `nvcf` namespace: +For `control` and `all`, the self-managed stack renders the resolved backend +settings directly into the function autoscaler chart's existing +`function-autoscaler-env` ConfigMap: ```yaml data: @@ -184,18 +180,19 @@ data: TIMESERIES_DB__IGNORE_ENV: "true" ``` -`autoscalerIntegration.namespace` sets the contract release namespace, and -`autoscalerIntegration.configMapName` overrides the autoscaler ConfigMap name. For an external backend, `metricsBackend.authentication.mode` supports `none`, `token`, and `mtls`. Token mode also requires `authnEndpoint`; mTLS mode requires paths for the mounted client certificate and private key. +The autoscaler chart owns this environment ConfigMap and its rollout checksum. +Its Vault Agent template remains a separate ConfigMap because it is mounted as +a file and has a separate lifecycle. There is no generic observability profile +or contract ConfigMap. + The self-managed Helmfile deploys the function autoscaler for `control` and -`all` and consumes this ConfigMap through the chart's external `envFrom` -reference. It also supplies the Cassandra and NVCF API endpoints needed by the +`all`. It also supplies the Cassandra and NVCF API endpoints needed by the self-managed runtime. The autoscaler does not need another observability enable -flag, backend mode, or duplicated VictoriaMetrics URL. The `compute` and -`disabled` profiles do not deploy it. +flag or backend mode. The `compute` and `disabled` profiles do not deploy it. ## Monitor Contracts @@ -207,6 +204,10 @@ Default monitors carry `nvcf.nvidia.com/observability-target: "true"` so the Target Allocator selects only NVCF-owned scrape targets. +The default monitor chart uses one generic ServiceMonitor template and one +generic PodMonitor template. Individual targets are data in chart values rather +than target-specific templates. + The default namespace is `monitoring`. If a deployment changes it, it must also provide NetworkPolicy reachability for the collector. diff --git a/deploy/stacks/observability/charts/nvcf-default-monitors/templates/dcgm-podmonitor.yaml b/deploy/stacks/observability/charts/nvcf-default-monitors/templates/podmonitors.yaml similarity index 51% rename from deploy/stacks/observability/charts/nvcf-default-monitors/templates/dcgm-podmonitor.yaml rename to deploy/stacks/observability/charts/nvcf-default-monitors/templates/podmonitors.yaml index 39f052b9e..6a8a6c7ac 100644 --- a/deploy/stacks/observability/charts/nvcf-default-monitors/templates/dcgm-podmonitor.yaml +++ b/deploy/stacks/observability/charts/nvcf-default-monitors/templates/podmonitors.yaml @@ -15,25 +15,36 @@ See the License for the specific language governing permissions and limitations under the License. */}} -{{- if and .Values.computePlane.enabled .Values.computePlane.dcgm.enabled }} +{{- $top := . }} +{{- $podMonitors := list + (dict "name" "dcgm" "values" .Values.computePlane.dcgm) + (dict "name" "worker" "values" .Values.computePlane.worker) }} +{{- if .Values.computePlane.enabled }} +{{- range $monitor := $podMonitors }} +{{- $name := get $monitor "name" }} +{{- $values := get $monitor "values" }} +{{- if $values.enabled }} +--- apiVersion: monitoring.coreos.com/v1 kind: PodMonitor metadata: - name: {{ include "nvcf-default-monitors.name" . }}-dcgm - namespace: {{ default .Release.Namespace .Values.computePlane.dcgm.namespace }} + name: {{ include "nvcf-default-monitors.name" $top }}-{{ $name }} + namespace: {{ default $top.Release.Namespace $values.namespace }} labels: - {{- include "nvcf-default-monitors.labels" . | nindent 4 }} - {{- with .Values.computePlane.dcgm.labels }} + {{- include "nvcf-default-monitors.labels" $top | nindent 4 }} + {{- with $values.labels }} {{- toYaml . | nindent 4 }} {{- end }} spec: - podMetricsEndpoints: - - interval: {{ .Values.computePlane.dcgm.interval | quote }} - path: {{ .Values.computePlane.dcgm.path | quote }} - port: {{ .Values.computePlane.dcgm.port | quote }} namespaceSelector: - {{- toYaml .Values.computePlane.dcgm.namespaceSelector | nindent 4 }} + {{- toYaml $values.namespaceSelector | nindent 4 }} selector: matchLabels: - nvca.nvcf.nvidia.io/dcgm-metrics-present: "true" + {{- toYaml $values.selector | nindent 6 }} + podMetricsEndpoints: + - interval: {{ $values.interval | quote }} + path: {{ $values.path | quote }} + port: {{ $values.port | quote }} +{{- end }} +{{- end }} {{- end }} diff --git a/deploy/stacks/observability/charts/nvcf-default-monitors/templates/worker-podmonitor.yaml b/deploy/stacks/observability/charts/nvcf-default-monitors/templates/worker-podmonitor.yaml deleted file mode 100644 index 3f168a2d0..000000000 --- a/deploy/stacks/observability/charts/nvcf-default-monitors/templates/worker-podmonitor.yaml +++ /dev/null @@ -1,38 +0,0 @@ -{{/* -SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -SPDX-License-Identifier: Apache-2.0 - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - https://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/}} - -{{- if and .Values.computePlane.enabled .Values.computePlane.worker.enabled }} -apiVersion: monitoring.coreos.com/v1 -kind: PodMonitor -metadata: - name: {{ include "nvcf-default-monitors.name" . }}-worker - namespace: {{ .Release.Namespace }} - labels: - {{- include "nvcf-default-monitors.labels" . | nindent 4 }} - {{- with .Values.computePlane.worker.labels }} - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - namespaceSelector: - {{- toYaml .Values.computePlane.worker.namespaceSelector | nindent 4 }} - selector: - matchLabels: {{ .Values.computePlane.worker.selector | toJson }} - podMetricsEndpoints: - - interval: {{ .Values.computePlane.worker.interval | quote }} - path: {{ .Values.computePlane.worker.path | quote }} - port: {{ .Values.computePlane.worker.port | quote }} -{{- end }} diff --git a/deploy/stacks/observability/charts/nvcf-default-monitors/values.yaml b/deploy/stacks/observability/charts/nvcf-default-monitors/values.yaml index 0bbdfd4c2..45a10b8ba 100644 --- a/deploy/stacks/observability/charts/nvcf-default-monitors/values.yaml +++ b/deploy/stacks/observability/charts/nvcf-default-monitors/values.yaml @@ -65,6 +65,8 @@ computePlane: namespace: "" namespaceSelector: any: true + selector: + nvca.nvcf.nvidia.io/dcgm-metrics-present: "true" interval: 30s path: /metrics # NVCA uses this name for the injected DCGM metrics port. @@ -73,6 +75,7 @@ computePlane: worker: enabled: true + namespace: "" namespaceSelector: matchExpressions: - key: nvca.nvcf.nvidia.io/workload-instance-type diff --git a/deploy/stacks/observability/charts/nvcf-observability-contract/Chart.yaml b/deploy/stacks/observability/charts/nvcf-observability-contract/Chart.yaml deleted file mode 100644 index deea259b2..000000000 --- a/deploy/stacks/observability/charts/nvcf-observability-contract/Chart.yaml +++ /dev/null @@ -1,21 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: v2 -name: nvcf-observability-contract -description: Resolved observability settings for NVCF stack consumers -type: application -version: 0.1.0 -appVersion: "0.1.0" diff --git a/deploy/stacks/observability/charts/nvcf-observability-contract/templates/autoscaler-configmap.yaml b/deploy/stacks/observability/charts/nvcf-observability-contract/templates/autoscaler-configmap.yaml deleted file mode 100644 index 5de41d5df..000000000 --- a/deploy/stacks/observability/charts/nvcf-observability-contract/templates/autoscaler-configmap.yaml +++ /dev/null @@ -1,42 +0,0 @@ -{{/* -SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -SPDX-License-Identifier: Apache-2.0 - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - https://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/}} - -{{- if .Values.autoscaler.enabled }} -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ .Values.autoscaler.configMapName }} - namespace: {{ .Values.autoscaler.namespace }} - labels: - app.kubernetes.io/name: nvcf-observability-contract - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: {{ .Release.Service }} - nvcf.nvidia.com/observability-profile: {{ .Values.profile | quote }} -data: - TIMESERIES_DB__TIMESERIES_DB_URL: {{ required "autoscaler.promqlEndpoint is required for control and all profiles" .Values.autoscaler.promqlEndpoint | quote }} - TIMESERIES_DB__AUTH_MODE: {{ .Values.autoscaler.authMode | quote }} - TIMESERIES_DB__IGNORE_ENV: {{ .Values.autoscaler.ignoreEnv | quote }} - {{- with .Values.autoscaler.authnEndpoint }} - TIMESERIES_DB__AUTHN_URL: {{ . | quote }} - {{- end }} - {{- with .Values.autoscaler.clientCertificatePath }} - TIMESERIES_DB__CLIENT_CERTIFICATE_PATH: {{ . | quote }} - {{- end }} - {{- with .Values.autoscaler.clientPrivateKeyPath }} - TIMESERIES_DB__CLIENT_PRIVATE_KEY_PATH: {{ . | quote }} - {{- end }} -{{- end }} diff --git a/deploy/stacks/observability/charts/nvcf-observability-contract/templates/profile-configmap.yaml b/deploy/stacks/observability/charts/nvcf-observability-contract/templates/profile-configmap.yaml deleted file mode 100644 index cb0f41309..000000000 --- a/deploy/stacks/observability/charts/nvcf-observability-contract/templates/profile-configmap.yaml +++ /dev/null @@ -1,36 +0,0 @@ -{{/* -SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -SPDX-License-Identifier: Apache-2.0 - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - https://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/}} - -{{- if ne .Values.profile "disabled" }} -apiVersion: v1 -kind: ConfigMap -metadata: - name: nvcf-observability-profile - namespace: {{ .Release.Namespace }} - labels: - app.kubernetes.io/name: nvcf-observability-contract - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: {{ .Release.Service }} - nvcf.nvidia.com/observability-profile: {{ .Values.profile | quote }} -data: - profile: {{ .Values.profile | quote }} - controlTargetsEnabled: {{ .Values.controlTargetsEnabled | quote }} - computeTargetsEnabled: {{ .Values.computeTargetsEnabled | quote }} - byooSupportEnabled: {{ .Values.byooSupportEnabled | quote }} - nvcaCollectorEnabled: {{ .Values.nvcaCollectorEnabled | quote }} - autoscalerIntegrationEnabled: {{ .Values.autoscaler.enabled | quote }} -{{- end }} diff --git a/deploy/stacks/observability/charts/nvcf-observability-contract/values.yaml b/deploy/stacks/observability/charts/nvcf-observability-contract/values.yaml deleted file mode 100644 index a7455fc30..000000000 --- a/deploy/stacks/observability/charts/nvcf-observability-contract/values.yaml +++ /dev/null @@ -1,32 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -profile: disabled - -controlTargetsEnabled: false -computeTargetsEnabled: false -byooSupportEnabled: false -nvcaCollectorEnabled: false - -autoscaler: - enabled: false - namespace: nvcf - configMapName: nvcf-observability-autoscaler - promqlEndpoint: "" - authMode: none - authnEndpoint: "" - clientCertificatePath: "" - clientPrivateKeyPath: "" - ignoreEnv: true diff --git a/deploy/stacks/observability/environments/base.yaml b/deploy/stacks/observability/environments/base.yaml index 6faf8d49e..a0907f5cf 100644 --- a/deploy/stacks/observability/environments/base.yaml +++ b/deploy/stacks/observability/environments/base.yaml @@ -27,13 +27,6 @@ metricsBackend: clientCertificatePath: "" clientPrivateKeyPath: "" -autoscalerIntegration: - namespace: nvcf - configMapName: nvcf-observability-autoscaler - # Self-hosted control-plane metrics do not require the hosted TSDB's - # environment label or authentication flow. - ignoreEnv: true - prometheusOperatorCrds: version: "30.0.0" values: @@ -168,12 +161,15 @@ defaultMonitors: enabled: true namespaceSelector: any: true + selector: + nvca.nvcf.nvidia.io/dcgm-metrics-present: "true" interval: 30s path: /metrics port: dcgm-metrics labels: {} worker: enabled: true + namespace: "" namespaceSelector: matchExpressions: - key: nvca.nvcf.nvidia.io/workload-instance-type diff --git a/deploy/stacks/observability/helmfile.d/01-observability.yaml.gotmpl b/deploy/stacks/observability/helmfile.d/01-observability.yaml.gotmpl index 83f1520d8..28be3c6fa 100644 --- a/deploy/stacks/observability/helmfile.d/01-observability.yaml.gotmpl +++ b/deploy/stacks/observability/helmfile.d/01-observability.yaml.gotmpl @@ -70,8 +70,6 @@ environments: {{- $discoveryRbacEnabled := eq $discoveryRbacMode "install" }} {{- $victoriaMetricsEnabled := eq $metricsBackendMode "install" }} {{- $defaultMonitorsEnabled := $profileEnabled }} -{{- $autoscalerIntegrationEnabled := $controlEnabled }} -{{- $contractNamespace := dig "autoscalerIntegration" "namespace" "nvcf" .Values }} {{- $victoriaMetricsNamespace := dig "victoriaMetrics" "namespace" $observabilityNamespace .Values }} {{- $bundledRemoteWriteEndpoint := printf "http://vmsingle.%s.svc.cluster.local:8428/api/v1/write" $victoriaMetricsNamespace }} {{- $bundledPromqlEndpoint := printf "http://vmsingle.%s.svc.cluster.local:8428" $victoriaMetricsNamespace }} @@ -241,31 +239,3 @@ releases: labels: release-group: observability {{- end }} - - {{- if $profileEnabled }} - - name: observability-contract - chart: ../charts/nvcf-observability-contract - namespace: {{ $contractNamespace }} - {{- if $victoriaMetricsEnabled }} - needs: - - {{ $victoriaMetricsNamespace }}/victoria-metrics - {{- end }} - values: - - profile: {{ $profile | quote }} - controlTargetsEnabled: {{ $controlEnabled }} - computeTargetsEnabled: {{ $computeEnabled }} - byooSupportEnabled: {{ $computeEnabled }} - nvcaCollectorEnabled: false - autoscaler: - enabled: {{ $autoscalerIntegrationEnabled }} - namespace: {{ dig "autoscalerIntegration" "namespace" "nvcf" .Values | quote }} - configMapName: {{ dig "autoscalerIntegration" "configMapName" "nvcf-observability-autoscaler" .Values | quote }} - promqlEndpoint: {{ $promqlEndpoint | quote }} - authMode: {{ $metricsBackendAuthMode | quote }} - authnEndpoint: {{ $metricsBackendAuthnEndpoint | quote }} - clientCertificatePath: {{ $metricsBackendClientCertificatePath | quote }} - clientPrivateKeyPath: {{ $metricsBackendClientPrivateKeyPath | quote }} - ignoreEnv: {{ dig "autoscalerIntegration" "ignoreEnv" true .Values }} - labels: - release-group: observability - {{- end }} diff --git a/deploy/stacks/observability/tests/profile-defaults.sh b/deploy/stacks/observability/tests/profile-defaults.sh index 7c4a0c968..94ec4273d 100755 --- a/deploy/stacks/observability/tests/profile-defaults.sh +++ b/deploy/stacks/observability/tests/profile-defaults.sh @@ -10,16 +10,6 @@ fail() { exit 1 } -assert_contract_value() { - local manifests="$1" - local profile="$2" - local key="$3" - local value="$4" - - grep -q "$key: \"$value\"" $manifests || - fail "$profile profile did not resolve $key to $value" -} - profile_releases() { local profile="$1" HELMFILE_ENV=local helmfile \ @@ -82,10 +72,18 @@ service_monitor_template_count="$( test "$service_monitor_template_count" = "1" || fail "default monitors chart must use one generic ServiceMonitor template" +pod_monitor_template_count="$( + find "$stack_dir/charts/nvcf-default-monitors/templates" \ + -maxdepth 1 -type f -name '*podmonitor*.yaml' | + wc -l | + tr -d ' ' +)" +test "$pod_monitor_template_count" = "1" || + fail "default monitors chart must use one generic PodMonitor template" + enabled_releases="$( cat <<'EOF' default-monitors -observability-contract opentelemetry-operator otel-collector prometheus-operator-crds @@ -165,54 +163,6 @@ if grep -q "nvcf-default-monitors-worker" $compute_worker_disabled_manifests; th fail "nested worker monitor override was ignored" fi -for profile in control compute all; do - HELMFILE_ENV=local helmfile \ - --file "$stack_dir/helmfile.d" \ - --environment default \ - --state-values-set "observability.profile=$profile" \ - --selector name=observability-contract \ - template --output-dir "$work_dir/$profile-contract" >/dev/null -done - -control_contract="$(find "$work_dir/control-contract" -type f -name '*.yaml' -print)" -compute_contract="$(find "$work_dir/compute-contract" -type f -name '*.yaml' -print)" -all_contract="$(find "$work_dir/all-contract" -type f -name '*.yaml' -print)" - -assert_contract_value "$control_contract" control controlTargetsEnabled true -assert_contract_value "$control_contract" control computeTargetsEnabled false -assert_contract_value "$control_contract" control byooSupportEnabled false -assert_contract_value "$control_contract" control nvcaCollectorEnabled false -assert_contract_value "$control_contract" control autoscalerIntegrationEnabled true - -assert_contract_value "$compute_contract" compute controlTargetsEnabled false -assert_contract_value "$compute_contract" compute computeTargetsEnabled true -assert_contract_value "$compute_contract" compute byooSupportEnabled true -assert_contract_value "$compute_contract" compute nvcaCollectorEnabled false -assert_contract_value "$compute_contract" compute autoscalerIntegrationEnabled false -if grep -q 'TIMESERIES_DB__TIMESERIES_DB_URL' $compute_contract; then - fail "compute profile published an autoscaler contract" -fi - -assert_contract_value "$all_contract" all controlTargetsEnabled true -assert_contract_value "$all_contract" all computeTargetsEnabled true -assert_contract_value "$all_contract" all byooSupportEnabled true -assert_contract_value "$all_contract" all nvcaCollectorEnabled false -assert_contract_value "$all_contract" all autoscalerIntegrationEnabled true - -HELMFILE_ENV=local helmfile \ - --file "$stack_dir/helmfile.d" \ - --environment default \ - --state-values-set observability.profile=control \ - --selector name=observability-contract \ - template --output-dir "$work_dir/bundled-contract" >/dev/null - -bundled_contract="$(find "$work_dir/bundled-contract" -type f -name '*.yaml' -print)" -grep -q 'TIMESERIES_DB__TIMESERIES_DB_URL: "http://vmsingle.monitoring.svc.cluster.local:8428"' \ - $bundled_contract || - fail "control profile did not publish the bundled PromQL endpoint" -grep -q 'TIMESERIES_DB__AUTH_MODE: "none"' $bundled_contract || - fail "control profile did not publish the bundled backend auth mode" - HELMFILE_ENV=local helmfile \ --file "$stack_dir/helmfile.d" \ --environment default \ @@ -229,43 +179,6 @@ external_manifests="$(find "$work_dir/external" -type f -name '*.yaml' -print)" grep -q 'endpoint: https://metrics.example.com/write' $external_manifests || fail "external remote-write endpoint was not applied to the collector" -HELMFILE_ENV=local helmfile \ - --file "$stack_dir/helmfile.d" \ - --environment default \ - --state-values-set observability.profile=control \ - --state-values-set observability.components.otelOperator.mode=existing \ - --state-values-set metricsBackend.mode=existing \ - --state-values-set metricsBackend.type=external \ - --state-values-set-string metricsBackend.remoteWriteEndpoint=https://metrics.example.com/write \ - --state-values-set-string metricsBackend.promqlEndpoint=https://metrics.example.com \ - --selector name=observability-contract \ - template --output-dir "$work_dir/external-contract" >/dev/null - -external_contract="$(find "$work_dir/external-contract" -type f -name '*.yaml' -print)" -grep -q 'TIMESERIES_DB__TIMESERIES_DB_URL: "https://metrics.example.com"' \ - $external_contract || - fail "external PromQL endpoint was not published for the autoscaler" - -HELMFILE_ENV=local helmfile \ - --file "$stack_dir/helmfile.d" \ - --environment default \ - --state-values-set observability.profile=control \ - --state-values-set observability.components.otelOperator.mode=existing \ - --state-values-set metricsBackend.mode=existing \ - --state-values-set metricsBackend.type=external \ - --state-values-set-string metricsBackend.remoteWriteEndpoint=https://metrics.example.com/write \ - --state-values-set-string metricsBackend.promqlEndpoint=https://metrics.example.com \ - --state-values-set metricsBackend.authentication.mode=token \ - --state-values-set-string metricsBackend.authentication.authnEndpoint=https://auth.example.com \ - --selector name=observability-contract \ - template --output-dir "$work_dir/token-contract" >/dev/null - -token_contract="$(find "$work_dir/token-contract" -type f -name '*.yaml' -print)" -grep -q 'TIMESERIES_DB__AUTH_MODE: "token"' $token_contract || - fail "token auth mode was not published for the autoscaler" -grep -q 'TIMESERIES_DB__AUTHN_URL: "https://auth.example.com"' $token_contract || - fail "token auth endpoint was not published for the autoscaler" - if HELMFILE_ENV=local helmfile \ --file "$stack_dir/helmfile.d" \ --environment default \ diff --git a/deploy/stacks/self-managed/environments/base.yaml b/deploy/stacks/self-managed/environments/base.yaml index 033a417a5..bfb1fd7f3 100644 --- a/deploy/stacks/self-managed/environments/base.yaml +++ b/deploy/stacks/self-managed/environments/base.yaml @@ -141,8 +141,7 @@ observability: # Defaults consumed only for the control and all observability profiles. functionAutoscaler: - # The first chart release containing external envFrom support. - chartVersion: "0.2.0" + chartVersion: "0.1.0" image: # Keep the stack's image pin independent of the chart's release metadata. tag: "1.18.3" @@ -155,6 +154,10 @@ functionAutoscaler: # Self-managed NVCF currently uses its internal unauthenticated gRPC path. disableAuth: true dryRun: false + timeseriesDb: + # Self-hosted control-plane metrics do not require the hosted TSDB's + # environment label. + ignoreEnv: true accounts: limits: diff --git a/deploy/stacks/self-managed/global.yaml.gotmpl b/deploy/stacks/self-managed/global.yaml.gotmpl index 49e2c6f53..b170f74e9 100644 --- a/deploy/stacks/self-managed/global.yaml.gotmpl +++ b/deploy/stacks/self-managed/global.yaml.gotmpl @@ -589,10 +589,14 @@ stateMetrics: serviceMonitor: enabled: false -{{- $observabilityContract := dict - "profile" (dig "observability" "profile" "disabled" .Values) - "metricsBackend" (dig "metricsBackend" dict .Values) - "autoscalerIntegration" (dig "autoscalerIntegration" dict .Values) }} +{{- $observabilityProfile := dig "observability" "profile" "disabled" .Values }} +{{- $functionAutoscalerEnabled := or (eq $observabilityProfile "control") (eq $observabilityProfile "all") }} +{{- if $functionAutoscalerEnabled }} +{{- $defaultMetricsBackendMode := ternary "install" "disabled" (ne $observabilityProfile "disabled") }} +{{- $metricsBackendMode := dig "metricsBackend" "mode" $defaultMetricsBackendMode .Values }} +{{- $victoriaMetricsNamespace := dig "victoriaMetrics" "namespace" (dig "observability" "namespace" "monitoring" .Values) .Values }} +{{- $bundledPromqlEndpoint := printf "http://vmsingle.%s.svc.cluster.local:8428" $victoriaMetricsNamespace }} +{{- $promqlEndpoint := ternary $bundledPromqlEndpoint (dig "metricsBackend" "promqlEndpoint" "" .Values) (eq $metricsBackendMode "install") }} functionautoscaler: namespace: nvcf fullnameOverride: function-autoscaler @@ -606,8 +610,6 @@ functionautoscaler: {{- with dig "functionAutoscaler" "image" "tag" nil .Values }} tag: {{ . | quote }} {{- end }} - podAnnotations: - checksum/observability-contract: {{ toJson $observabilityContract | sha256sum | quote }} env: CONFIG: /etc/server/config/settings-local.yaml SECRETS_PATH: /vault/secrets/secrets.json @@ -617,15 +619,25 @@ functionautoscaler: NVCF_API__NVCF_API_GRPC_ADDRESS: {{ dig "functionAutoscaler" "nvcfApi" "grpcAddress" "http://api.nvcf.svc.cluster.local:9090" .Values | quote }} NVCF_API__DISABLE_AUTH: {{ dig "functionAutoscaler" "nvcfApi" "disableAuth" true .Values | quote }} NVCF_API__DRY_RUN: {{ dig "functionAutoscaler" "nvcfApi" "dryRun" false .Values | quote }} - envFrom: - - configMapRef: - name: {{ dig "autoscalerIntegration" "configMapName" "nvcf-observability-autoscaler" .Values }} + TIMESERIES_DB__TIMESERIES_DB_URL: {{ required "metricsBackend.promqlEndpoint is required for control and all profiles" $promqlEndpoint | quote }} + TIMESERIES_DB__AUTH_MODE: {{ dig "metricsBackend" "authentication" "mode" "none" .Values | quote }} + TIMESERIES_DB__IGNORE_ENV: {{ dig "functionAutoscaler" "timeseriesDb" "ignoreEnv" true .Values | quote }} + {{- with dig "metricsBackend" "authentication" "authnEndpoint" "" .Values }} + TIMESERIES_DB__AUTHN_URL: {{ . | quote }} + {{- end }} + {{- with dig "metricsBackend" "authentication" "clientCertificatePath" "" .Values }} + TIMESERIES_DB__CLIENT_CERTIFICATE_PATH: {{ . | quote }} + {{- end }} + {{- with dig "metricsBackend" "authentication" "clientPrivateKeyPath" "" .Values }} + TIMESERIES_DB__CLIENT_PRIVATE_KEY_PATH: {{ . | quote }} + {{- end }} {{- with include "nvcf.nodeSelector" (dict "type" "controlplane" "selectors" .Values.global.nodeSelectors) }} {{- . | nindent 2 }} {{- end }} {{- with include "nvcf.tolerations" (dict "type" "controlplane" "tolerations" .Values.global.tolerations) }} {{- . | nindent 2 }} {{- end }} +{{- end }} reval: fullnameOverride: reval diff --git a/deploy/stacks/self-managed/helmfile.d/00-observability-infrastructure.yaml.gotmpl b/deploy/stacks/self-managed/helmfile.d/00-observability-infrastructure.yaml.gotmpl index 6f69b5ff4..9e50d3c41 100644 --- a/deploy/stacks/self-managed/helmfile.d/00-observability-infrastructure.yaml.gotmpl +++ b/deploy/stacks/self-managed/helmfile.d/00-observability-infrastructure.yaml.gotmpl @@ -20,7 +20,7 @@ helmfiles: {{- toYaml (dig "global" (dict) .Values) | nindent 10 }} - observability: {{- toYaml $observability | nindent 10 }} - {{- range $key := list "metricsBackend" "autoscalerIntegration" "prometheusOperatorCrds" "opentelemetryOperator" "collector" "victoriaMetrics" "defaultMonitors" }} + {{- range $key := list "metricsBackend" "prometheusOperatorCrds" "opentelemetryOperator" "collector" "victoriaMetrics" "defaultMonitors" }} {{- with (index $.Values $key) }} - {{ $key }}: {{- toYaml . | nindent 10 }} diff --git a/deploy/stacks/self-managed/helmfile.d/03-observability.yaml.gotmpl b/deploy/stacks/self-managed/helmfile.d/03-observability.yaml.gotmpl index 8ed7a655a..f8654778d 100644 --- a/deploy/stacks/self-managed/helmfile.d/03-observability.yaml.gotmpl +++ b/deploy/stacks/self-managed/helmfile.d/03-observability.yaml.gotmpl @@ -65,7 +65,7 @@ releases: {{- if $functionAutoscalerEnabled }} - name: function-autoscaler - version: {{ dig "functionAutoscaler" "chartVersion" "0.2.0" .Values | quote }} + version: {{ dig "functionAutoscaler" "chartVersion" "0.1.0" .Values | quote }} namespace: nvcf inherit: - template: functionAutoscaler diff --git a/deploy/stacks/self-managed/tests/observability-autoscaler.sh b/deploy/stacks/self-managed/tests/observability-autoscaler.sh index 38b599703..1001a4237 100755 --- a/deploy/stacks/self-managed/tests/observability-autoscaler.sh +++ b/deploy/stacks/self-managed/tests/observability-autoscaler.sh @@ -36,18 +36,6 @@ release_count() { awk -v release="$release" 'NR > 1 && $1 == release {count++} END {print count + 0}' } -release_namespace() { - local profile="$1" - local release="$2" - - HELMFILE_ENV=base helmfile \ - "${helmfile_args[@]}" \ - "${state_values[@]}" \ - --state-values-set "observability.profile=$profile" \ - list 2>/dev/null | - awk -v release="$release" 'NR > 1 && $1 == release {print $2}' -} - for profile in control compute all; do test "$(release_count "$profile" victoria-metrics)" = "1" || fail "$profile profile did not install exactly one shared metrics backend" @@ -55,8 +43,6 @@ done test "$(release_count disabled victoria-metrics)" = "0" || fail "disabled profile installed the shared metrics backend" -test "$(release_namespace control observability-contract)" = "nvcf" || - fail "control profile did not place the contract in the autoscaler namespace" test "$(release_count control function-autoscaler)" = "1" || fail "control profile did not install exactly one function autoscaler" test "$(release_count all function-autoscaler)" = "1" || @@ -66,14 +52,22 @@ test "$(release_count compute function-autoscaler)" = "0" || test "$(release_count disabled function-autoscaler)" = "0" || fail "disabled profile installed the function autoscaler" -HELMFILE_ENV=base HELMFILE_CACHE_HOME="$work_dir/helmfile-cache" helmfile \ - --file "$stack_dir/helmfile.d/03-observability.yaml.gotmpl" \ - --environment default \ - "${state_values[@]}" \ - --state-values-set observability.profile=control \ - --selector name=function-autoscaler \ - write-values \ - --output-file-template "$work_dir/autoscaler-values.yaml" >/dev/null +write_autoscaler_values() { + local output_file="$1" + shift + + HELMFILE_ENV=base HELMFILE_CACHE_HOME="$work_dir/helmfile-cache" helmfile \ + --file "$stack_dir/helmfile.d/03-observability.yaml.gotmpl" \ + --environment default \ + "${state_values[@]}" \ + --state-values-set observability.profile=control \ + "$@" \ + --selector name=function-autoscaler \ + write-values \ + --output-file-template "$output_file" >/dev/null +} + +write_autoscaler_values "$work_dir/autoscaler-values.yaml" autoscaler_values="$work_dir/autoscaler-values.yaml" for expected in \ @@ -82,7 +76,9 @@ for expected in \ 'NVCF_API__NVCF_API_GRPC_ADDRESS: http://api.nvcf.svc.cluster.local:9090' \ 'NVCF_API__DISABLE_AUTH: "true"' \ 'NVCF_API__DRY_RUN: "false"' \ - 'name: nvcf-observability-autoscaler'; do + 'TIMESERIES_DB__TIMESERIES_DB_URL: http://vmsingle.monitoring.svc.cluster.local:8428' \ + 'TIMESERIES_DB__AUTH_MODE: none' \ + 'TIMESERIES_DB__IGNORE_ENV: "true"'; do grep -q "$expected" "$autoscaler_values" || fail "control profile did not render autoscaler value: $expected" done @@ -95,13 +91,39 @@ helm template function-autoscaler "$repo_dir/deploy/helm/function-autoscaler" \ autoscaler_manifests="$work_dir/autoscaler-manifests.yaml" grep -q 'image: nvcr.io/YOUR_ORG/YOUR_TEAM/nvcf-function-autoscaler:1.18.3' "$autoscaler_manifests" || fail "self-managed stack did not pin the autoscaler image" -test "$(grep -c 'name: nvcf-observability-autoscaler' "$autoscaler_manifests")" = "1" || - fail "autoscaler chart did not render exactly one external observability envFrom" +test "$(grep -c '^kind: ConfigMap$' "$autoscaler_manifests")" = "2" || + fail "autoscaler chart did not render only its env and Vault template ConfigMaps" +grep -q 'name: function-autoscaler-env' "$autoscaler_manifests" || + fail "autoscaler chart did not render its chart-owned env ConfigMap" +if grep -q 'nvcf-observability-autoscaler\|nvcf-observability-profile' "$autoscaler_manifests"; then + fail "autoscaler chart rendered a redundant observability contract ConfigMap" +fi grep -q 'NVCF_API__DRY_RUN: "false"' "$autoscaler_manifests" || fail "autoscaler chart did not render the self-managed runtime configuration" +grep -q 'TIMESERIES_DB__TIMESERIES_DB_URL: "http://vmsingle.monitoring.svc.cluster.local:8428"' \ + "$autoscaler_manifests" || + fail "autoscaler chart did not consolidate the bundled PromQL endpoint into its env ConfigMap" grep -q '"helm.sh/hook": test' "$autoscaler_manifests" || fail "autoscaler chart lost its runtime helm test hook" +write_autoscaler_values "$work_dir/external-autoscaler-values.yaml" \ + --state-values-set metricsBackend.mode=existing \ + --state-values-set metricsBackend.type=external \ + --state-values-set-string metricsBackend.promqlEndpoint=https://metrics.example.com \ + --state-values-set metricsBackend.authentication.mode=token \ + --state-values-set-string metricsBackend.authentication.authnEndpoint=https://auth.example.com \ + --state-values-set functionAutoscaler.timeseriesDb.ignoreEnv=false + +external_autoscaler_values="$work_dir/external-autoscaler-values.yaml" +for expected in \ + 'TIMESERIES_DB__TIMESERIES_DB_URL: https://metrics.example.com' \ + 'TIMESERIES_DB__AUTH_MODE: token' \ + 'TIMESERIES_DB__AUTHN_URL: https://auth.example.com' \ + 'TIMESERIES_DB__IGNORE_ENV: "false"'; do + grep -q "$expected" "$external_autoscaler_values" || + fail "external backend did not render autoscaler value: $expected" +done + if HELMFILE_ENV=base helmfile \ "${helmfile_args[@]}" \ "${state_values[@]}" \ From 669d2afae686d9d38778d69ea69ec071af578d45 Mon Sep 17 00:00:00 2001 From: Kristina Pathak Date: Tue, 28 Jul 2026 14:29:13 -0700 Subject: [PATCH 09/13] fix(observability): address review feedback --- .../dev/skills/nvcf-explore-stack/SKILL.md | 2 +- deploy/stacks/observability/AGENTS.md | 21 ++ deploy/stacks/observability/README.md | 198 +++++------------- .../templates/podmonitors.yaml | 3 +- .../charts/nvcf-default-monitors/values.yaml | 8 +- .../observability/environments/base.yaml | 12 +- .../observability/tests/profile-defaults.sh | 12 ++ .../tests/observability-autoscaler.sh | 17 ++ 8 files changed, 115 insertions(+), 158 deletions(-) diff --git a/ai-tooling/dev/skills/nvcf-explore-stack/SKILL.md b/ai-tooling/dev/skills/nvcf-explore-stack/SKILL.md index 098a1ce8f..f8d1b7630 100644 --- a/ai-tooling/dev/skills/nvcf-explore-stack/SKILL.md +++ b/ai-tooling/dev/skills/nvcf-explore-stack/SKILL.md @@ -71,7 +71,7 @@ Walk me through the full deployment order : Summarize control-plane stages 0 through 3 from the self-managed gotmpl files. Stage 0 delegates to the shared observability Helmfile when `observability.profile` is enabled. Stage 3 installs State Metrics and the function autoscaler for `control` and `all`. Then summarize the compute-plane stage from `deploy/stacks/nvcf-compute-plane/helmfile.d/01-dependencies.yaml.gotmpl` and `deploy/stacks/nvcf-compute-plane/helmfile.d/02-nvca.yaml.gotmpl`. Call out which releases run in parallel inside a stage and which are serialized by `needs:`. Which subtree do I edit to change X -: Point at the Helmfile path for orchestration, `deploy/helm//` for checked-in chart wiring, and the chart image repository under `src/`, `infra/`, or `migrations/` for runtime behavior. All stack sources are monorepo-native, so edits land in this repository. +: Point at the Helmfile path for orchestration, `deploy/helm//` for checked-in chart wiring, and the image source under `src/`, `infra/`, or `migrations/` for runtime behavior. If a referenced chart is not checked in, report its Helmfile chart reference and do not claim a local source path. What namespaces does the stack use : Return the list from the helmfile (`namespace:` per release). diff --git a/deploy/stacks/observability/AGENTS.md b/deploy/stacks/observability/AGENTS.md index 94184d524..d71853a8c 100644 --- a/deploy/stacks/observability/AGENTS.md +++ b/deploy/stacks/observability/AGENTS.md @@ -40,3 +40,24 @@ deployments. Do not use Helm `lookup` to silently change profiles or component ownership based on live cluster state. The consuming stack must choose the profile explicitly. + +## Development + +- Render the local environment with `make template HELMFILE_ENV=local`. +- Run profile and integration assertions with `make test`. +- Run `git diff --check` before committing. + +## Code Style + +- Keep profile derivation and validation at the top of the Helmfile. +- Keep chart templates data-driven; use one generic template per monitor kind. +- Put stable scrape endpoints, labels, ports, and paths in the application + chart that exposes them. + +## Subtree Conventions + +- Keep reusable defaults in `environments/base.yaml`; environment files should + contain only overrides. +- Preserve explicit user values when applying profile defaults. +- Validate every new mode, dependency, and rendered monitor shape in + `tests/profile-defaults.sh`. diff --git a/deploy/stacks/observability/README.md b/deploy/stacks/observability/README.md index 5c486f486..b4fa54501 100644 --- a/deploy/stacks/observability/README.md +++ b/deploy/stacks/observability/README.md @@ -1,85 +1,34 @@ -# nvcf-observability-stack +# NVCF observability stack -Reusable Helmfile stack for self-hosted NVCF observability. A cluster installs -this stack at most once and selects the targets with one value: +Install this stack at most once per cluster. One profile selects what it +observes: ```yaml observability: profile: control ``` -The stack can own: - -- Prometheus Operator CRDs for `ServiceMonitor` and `PodMonitor` -- OpenTelemetry Operator -- One OpenTelemetry Collector with Target Allocator support -- Read-only discovery RBAC -- VictoriaMetrics -- NVCF-owned default monitor resources - ## Profiles -| Profile | Control targets | Compute targets | Result | -| --- | --- | --- | --- | -| `disabled` | No | No | Render no observability releases or resources. | -| `control` | Yes | No | Install one stack for control-plane metrics and the autoscaler backend. | -| `compute` | No | Yes | Install one stack for NVCA, DCGM, and worker metrics; resolve NVCA BYOO support on. | -| `all` | Yes | Yes | Install the union once; do not duplicate shared components. | - -The packaged reusable stack defaults to `disabled`. Its self-managed -control-plane example overlay defaults to `control`. A standalone compute-plane -consumer should select `compute`. A colocated deployment should install this -artifact once with `all`; it must not configure a second observability-stack -release from the compute-plane consumer. - -The public configuration does not expose -`planes.control.enabled` or `planes.compute.enabled`. Those values are derived -internally from the profile. - -## Profile Defaults - -Every enabled profile installs these shared components by default: - -- Prometheus Operator CRDs -- OpenTelemetry Operator -- One OpenTelemetry Collector -- Target Allocator -- Discovery RBAC -- One VictoriaMetrics instance +| Profile | Defaults | +| --- | --- | +| `disabled` | Render nothing. | +| `control` | Install shared infrastructure, control-plane monitors, and the function autoscaler backend. | +| `compute` | Install shared infrastructure and NVCA, DCGM, and worker monitors. | +| `all` | Install the union of `control` and `compute`, with shared components only once. | -The profile-specific defaults are: +Enabled profiles install Prometheus Operator CRDs, the OpenTelemetry Operator, +one collector with Target Allocator and discovery RBAC, VictoriaMetrics, and +the selected monitors. The reusable stack defaults to `disabled`; the +self-managed stack defaults to `control`. -| Default | `disabled` | `control` | `compute` | `all` | -| --- | --- | --- | --- | --- | -| Control-plane monitors | Off | On | Off | On | -| NVCA `ServiceMonitor` | Off | Off | On | On | -| DCGM `PodMonitor` | Off | Off | On | On | -| Worker `PodMonitor` | Off | Off | On | On | -| BYOO support | Off | Off | On | On | -| Autoscaler integration | Off | On | Off | On | -| NVCA collector | Off | Off | Off | Off | +Profiles derive the plane behavior internally. There are no +`planes.control.enabled` or `planes.compute.enabled` values. -Selecting a plane defaults its complete monitor set on. No separate -plane-monitor value is required for the normal path, and `all` defaults both -sets on. Explicit `defaultMonitors.controlPlane.enabled` and -`defaultMonitors.computePlane.enabled` values override the profile defaults. +## Overrides -Control-plane monitoring covers State Metric Service, Invocation Service, gRPC -Proxy, and LLM API Gateway. Compute monitoring selects NVCA in `nvca-system`, -DCGM pods that carry NVCA's DCGM metrics label, and pods exposing a `metrics` -port in NVCA-managed workload namespaces. - -For `compute` and `all`, the profile defaults BYOO support on. -The NVCA installer should map that value to its existing `BYOObservability` -feature gate. The feature gate enables the per-function BYOO collector path; it -does not create a second shared collector. This stack does not deploy NVCA. - -## Fine-Grained Overrides - -Profiles provide normal defaults. Advanced installations can override shared -component ownership with `install`, `existing`, or `disabled`. - -Monitor groups can also be overridden without changing the selected profile: +Profiles are defaults, not restrictions. Monitor groups and individual targets +remain configurable: ```yaml observability: @@ -94,15 +43,15 @@ defaultMonitors: enabled: false ``` -These values control only monitor rendering. They do not change the selected -profile, BYOO support, or autoscaler integration. The `disabled` profile -still omits the entire observability release. Compute monitor overrides live -under `defaultMonitors.computePlane.services` for ServiceMonitors, -`defaultMonitors.computePlane.dcgm`, and -`defaultMonitors.computePlane.worker`. +The component modes are `install`, `existing`, and `disabled`: -The fully expanded ownership configuration below is equivalent to setting only -`observability.profile: control`: +| Mode | Meaning | +| --- | --- | +| `install` | This stack installs and owns the component. | +| `existing` | Skip installation; the deployment workflow must verify a compatible component. | +| `disabled` | Do not install or use the component. | + +This expanded configuration is equivalent to `profile: control`: ```yaml observability: @@ -124,8 +73,7 @@ metricsBackend: type: victoriaMetrics ``` -For example, a deployment that supplies its own operator and metrics backend -can override only those ownership decisions: +For customer-managed infrastructure, override only its owners: ```yaml observability: @@ -141,90 +89,46 @@ metricsBackend: promqlEndpoint: https://metrics.example.com ``` -Supported component paths are: - -- `observability.components.prometheusOperatorCrds.mode` -- `observability.components.otelOperator.mode` -- `observability.components.collector.mode` -- `observability.components.targetAllocator.mode` -- `observability.components.discoveryRbac.mode` -- `metricsBackend.mode` +Invalid dependency combinations fail during Helmfile rendering. A disabled +profile cannot install individual components. -`install` makes this stack the owner. `existing` skips installation and requires -a separate installer preflight to verify a compatible external component. -`disabled` does not install or use the component. Overrides that contradict -required dependencies fail during Helmfile rendering. Component overrides -cannot install resources under `profile: disabled`. +## Metrics backend and autoscaler -The bundled backend derives both endpoints from the VictoriaMetrics namespace: +The bundled VictoriaMetrics endpoints are: ```text remote write: http://vmsingle.monitoring.svc.cluster.local:8428/api/v1/write PromQL: http://vmsingle.monitoring.svc.cluster.local:8428 ``` -An external backend must provide both endpoints for `control` and `all`. +An external backend always requires `remoteWriteEndpoint`; `control` and `all` +also require `promqlEndpoint`. Authentication modes are `none`, `token`, and +`mtls`. Token mode requires `authnEndpoint`; mTLS requires +`clientCertificatePath` and `clientPrivateKeyPath`. -## Autoscaler Integration +The function autoscaler is a self-managed control-plane component, not part of +this shared stack. For `control` and `all`, the self-managed Helmfile passes the +resolved PromQL and authentication values into the autoscaler chart's +`function-autoscaler-env` ConfigMap. `compute` and `disabled` do not deploy the +autoscaler. -The autoscaler is a control-plane consumer, not a second observability stack. -It reads from the same `metricsBackend.promqlEndpoint` selected by the profile. -For `control` and `all`, the self-managed stack renders the resolved backend -settings directly into the function autoscaler chart's existing -`function-autoscaler-env` ConfigMap: +## Monitor ownership -```yaml -data: - TIMESERIES_DB__TIMESERIES_DB_URL: http://vmsingle.monitoring.svc.cluster.local:8428 - TIMESERIES_DB__AUTH_MODE: none - TIMESERIES_DB__IGNORE_ENV: "true" -``` - -For an external backend, `metricsBackend.authentication.mode` supports `none`, -`token`, and `mtls`. Token mode also requires `authnEndpoint`; mTLS mode -requires paths for the mounted client certificate and private key. - -The autoscaler chart owns this environment ConfigMap and its rollout checksum. -Its Vault Agent template remains a separate ConfigMap because it is mounted as -a file and has a separate lifecycle. There is no generic observability profile -or contract ConfigMap. - -The self-managed Helmfile deploys the function autoscaler for `control` and -`all`. It also supplies the Cassandra and NVCF API endpoints needed by the -self-managed runtime. The autoscaler does not need another observability enable -flag or backend mode. The `compute` and `disabled` profiles do not deploy it. - -## Monitor Contracts - -Application charts own stable metrics endpoints, labels, named ports, paths, -and namespaces. This stack owns the default monitor resources that select those -contracts. - -Default monitors carry -`nvcf.nvidia.com/observability-target: "true"` so the Target Allocator selects -only NVCF-owned scrape targets. +Application charts own metrics ports, paths, labels, and namespaces. This stack +owns the `ServiceMonitor` and `PodMonitor` resources. One generic template per +kind renders all targets from values. -The default monitor chart uses one generic ServiceMonitor template and one -generic PodMonitor template. Individual targets are data in chart values rather -than target-specific templates. +All default monitors carry +`nvcf.nvidia.com/observability-target: "true"` for Target Allocator discovery. +Compute defaults select NVCA in `nvca-system`, DCGM pods by their NVCA metrics +label, and NVCA-managed workload pods by `icms-request-id`. -The default namespace is `monitoring`. If a deployment changes it, it must also -provide NetworkPolicy reachability for the collector. +The default observability namespace is `monitoring`. A different namespace +also requires NetworkPolicy reachability to the collector. -## Local Rendering - -The checked-in local environment is disabled by default: +## Validate ```sh make template HELMFILE_ENV=local -``` - -Change only `observability.profile` in the environment file to render a -complete enabled profile. Replace the example chart repository and image -repository before installing in a cluster. - -Run the Helmfile profile assertions and autoscaler chart render checks with: - -```sh make test ``` diff --git a/deploy/stacks/observability/charts/nvcf-default-monitors/templates/podmonitors.yaml b/deploy/stacks/observability/charts/nvcf-default-monitors/templates/podmonitors.yaml index 6a8a6c7ac..c5efa01ec 100644 --- a/deploy/stacks/observability/charts/nvcf-default-monitors/templates/podmonitors.yaml +++ b/deploy/stacks/observability/charts/nvcf-default-monitors/templates/podmonitors.yaml @@ -39,8 +39,7 @@ spec: namespaceSelector: {{- toYaml $values.namespaceSelector | nindent 4 }} selector: - matchLabels: - {{- toYaml $values.selector | nindent 6 }} + {{- toYaml $values.selector | nindent 4 }} podMetricsEndpoints: - interval: {{ $values.interval | quote }} path: {{ $values.path | quote }} diff --git a/deploy/stacks/observability/charts/nvcf-default-monitors/values.yaml b/deploy/stacks/observability/charts/nvcf-default-monitors/values.yaml index 45a10b8ba..9e47a8d24 100644 --- a/deploy/stacks/observability/charts/nvcf-default-monitors/values.yaml +++ b/deploy/stacks/observability/charts/nvcf-default-monitors/values.yaml @@ -66,7 +66,8 @@ computePlane: namespaceSelector: any: true selector: - nvca.nvcf.nvidia.io/dcgm-metrics-present: "true" + matchLabels: + nvca.nvcf.nvidia.io/dcgm-metrics-present: "true" interval: 30s path: /metrics # NVCA uses this name for the injected DCGM metrics port. @@ -77,11 +78,12 @@ computePlane: enabled: true namespace: "" namespaceSelector: + any: true + selector: matchExpressions: - - key: nvca.nvcf.nvidia.io/workload-instance-type + - key: icms-request-id operator: Exists interval: 30s path: /metrics port: metrics - selector: {} labels: {} diff --git a/deploy/stacks/observability/environments/base.yaml b/deploy/stacks/observability/environments/base.yaml index a0907f5cf..e45585972 100644 --- a/deploy/stacks/observability/environments/base.yaml +++ b/deploy/stacks/observability/environments/base.yaml @@ -159,9 +159,10 @@ defaultMonitors: labels: {} dcgm: enabled: true - namespaceSelector: - any: true - selector: + namespaceSelector: + any: true + selector: + matchLabels: nvca.nvcf.nvidia.io/dcgm-metrics-present: "true" interval: 30s path: /metrics @@ -171,11 +172,12 @@ defaultMonitors: enabled: true namespace: "" namespaceSelector: + any: true + selector: matchExpressions: - - key: nvca.nvcf.nvidia.io/workload-instance-type + - key: icms-request-id operator: Exists interval: 30s path: /metrics port: metrics - selector: {} labels: {} diff --git a/deploy/stacks/observability/tests/profile-defaults.sh b/deploy/stacks/observability/tests/profile-defaults.sh index 94ec4273d..a0ec2524c 100755 --- a/deploy/stacks/observability/tests/profile-defaults.sh +++ b/deploy/stacks/observability/tests/profile-defaults.sh @@ -120,6 +120,18 @@ compute_swapped_manifests="$(find "$work_dir/compute-swapped" -type f -name '*.y compute_worker_disabled_manifests="$(find "$work_dir/compute-worker-disabled" -type f -name '*.yaml' -print)" chart_control_manifests="$work_dir/chart-control-defaults.yaml" chart_compute_manifests="$work_dir/chart-compute-defaults.yaml" +worker_monitor_manifest="$work_dir/chart-worker-podmonitor.yaml" + +sed -n '/name: nvcf-default-monitors-worker/,$p' \ + "$chart_compute_manifests" >"$worker_monitor_manifest" +grep -q '^ any: true$' "$worker_monitor_manifest" || + fail "worker PodMonitor must use a supported all-namespaces selector" +grep -q '^ matchExpressions:$' "$worker_monitor_manifest" || + fail "worker PodMonitor must use a pod label expression" +grep -q '^ - key: icms-request-id$' "$worker_monitor_manifest" || + fail "worker PodMonitor must select NVCA-managed workload pods" +grep -q '^ operator: Exists$' "$worker_monitor_manifest" || + fail "worker PodMonitor label expression must use Exists" for monitor in state-metrics invocation-service grpc-proxy llm-api-gateway; do grep -q "nvcf-default-monitors-$monitor" "$chart_control_manifests" || diff --git a/deploy/stacks/self-managed/tests/observability-autoscaler.sh b/deploy/stacks/self-managed/tests/observability-autoscaler.sh index 1001a4237..4cf17c4c0 100755 --- a/deploy/stacks/self-managed/tests/observability-autoscaler.sh +++ b/deploy/stacks/self-managed/tests/observability-autoscaler.sh @@ -124,6 +124,23 @@ for expected in \ fail "external backend did not render autoscaler value: $expected" done +write_autoscaler_values "$work_dir/mtls-autoscaler-values.yaml" \ + --state-values-set metricsBackend.mode=existing \ + --state-values-set metricsBackend.type=external \ + --state-values-set-string metricsBackend.promqlEndpoint=https://metrics.example.com \ + --state-values-set metricsBackend.authentication.mode=mtls \ + --state-values-set-string metricsBackend.authentication.clientCertificatePath=/tls/client.crt \ + --state-values-set-string metricsBackend.authentication.clientPrivateKeyPath=/tls/client.key + +mtls_autoscaler_values="$work_dir/mtls-autoscaler-values.yaml" +for expected in \ + 'TIMESERIES_DB__AUTH_MODE: mtls' \ + 'TIMESERIES_DB__CLIENT_CERTIFICATE_PATH: /tls/client.crt' \ + 'TIMESERIES_DB__CLIENT_PRIVATE_KEY_PATH: /tls/client.key'; do + grep -q "$expected" "$mtls_autoscaler_values" || + fail "mTLS backend did not render autoscaler value: $expected" +done + if HELMFILE_ENV=base helmfile \ "${helmfile_args[@]}" \ "${state_values[@]}" \ From 427d5880a0c115c8e10e28d35bac58e581c0308d Mon Sep 17 00:00:00 2001 From: Kristina Pathak Date: Tue, 28 Jul 2026 14:44:16 -0700 Subject: [PATCH 10/13] fix(observability): wire compute profile to nvca --- deploy/stacks/nvcf-compute-plane/Makefile | 10 +- deploy/stacks/nvcf-compute-plane/README.md | 25 +++++ .../nvcf-compute-plane/environments/base.yaml | 4 + .../helmfile.d/02-nvca.yaml.gotmpl | 24 +++- .../self-managed-nvcfbackend-cm.yaml | 4 +- .../tests/compare-golden.sh | 27 +++++ .../tests/observability-profile.sh | 105 ++++++++++++++++++ deploy/stacks/observability/AGENTS.md | 4 +- deploy/stacks/observability/Makefile | 1 + deploy/stacks/observability/README.md | 4 + 10 files changed, 199 insertions(+), 9 deletions(-) create mode 100755 deploy/stacks/nvcf-compute-plane/tests/compare-golden.sh create mode 100755 deploy/stacks/nvcf-compute-plane/tests/observability-profile.sh diff --git a/deploy/stacks/nvcf-compute-plane/Makefile b/deploy/stacks/nvcf-compute-plane/Makefile index 40991e3c7..a1da937b5 100644 --- a/deploy/stacks/nvcf-compute-plane/Makefile +++ b/deploy/stacks/nvcf-compute-plane/Makefile @@ -60,7 +60,7 @@ include Makefile.dist -include helmfile-docker.mk # --- Development-Only Targets --- -.PHONY: dist clean-dist ensure-helm ensure-helmfile ensure-binaries render-local test-local generate-golden +.PHONY: dist clean-dist ensure-helm ensure-helmfile ensure-binaries render-local test-observability-profile test-local generate-golden # --- Binary Management (Development Only) --- ensure-helm: @@ -159,9 +159,12 @@ render-local: dist @cp -r testdata/registration $(DIST_DIR)/ @cd $(DIST_DIR) && CLUSTER_NAME=ncp-local NCA_ID=ncp-local HELMFILE_ENV=local make template -test-local: render-local +test-observability-profile: + @tests/observability-profile.sh + +test-local: test-observability-profile render-local @echo ">>> Comparing $(DIST_DIR)/out against golden $(GOLDEN_LOCAL_DIR)..." - @if ! diff -ruN "$(GOLDEN_LOCAL_DIR)" "$(DIST_DIR)/out"; then \ + @if ! tests/compare-golden.sh "$(GOLDEN_LOCAL_DIR)" "$(DIST_DIR)/out"; then \ echo ""; \ echo "ERROR: rendered output in $(DIST_DIR)/out differs from golden $(GOLDEN_LOCAL_DIR)."; \ echo " Review the diff above. If the change is intentional, refresh the golden files:"; \ @@ -208,6 +211,7 @@ help: @echo " clean-dist Remove the distribution directory" @echo "" @echo "Testing Targets:" + @echo " test-observability-profile Verify NVCA observability profile defaults" @echo " test-local Render the local env and diff against golden testdata" @echo " generate-golden Render the local env and update golden testdata" @echo "" diff --git a/deploy/stacks/nvcf-compute-plane/README.md b/deploy/stacks/nvcf-compute-plane/README.md index ce99c15cf..d0ec1639b 100644 --- a/deploy/stacks/nvcf-compute-plane/README.md +++ b/deploy/stacks/nvcf-compute-plane/README.md @@ -40,6 +40,31 @@ Helmfile. an environment file for your registry and service endpoints, then pass its name without the `.yaml` suffix. +## Observability + +The stack defaults `observability.profile` to `compute`. The `compute` and +`all` profiles enable the NVCA collector and `BYOObservability` feature gate. +The `control` and `disabled` profiles leave both off. + +One value selects the normal behavior: + +```yaml +observability: + profile: compute +``` + +Explicit NVCA values override the profile defaults: + +```yaml +global: + nvcaOperator: + selfManaged: + otelCollector: + enabled: false + featureGateValues: + - "-BYOObservability" +``` + ## Chart and Image Sources The stack pins the NVCA operator chart in diff --git a/deploy/stacks/nvcf-compute-plane/environments/base.yaml b/deploy/stacks/nvcf-compute-plane/environments/base.yaml index 56175a959..1062383ee 100644 --- a/deploy/stacks/nvcf-compute-plane/environments/base.yaml +++ b/deploy/stacks/nvcf-compute-plane/environments/base.yaml @@ -63,6 +63,10 @@ global: # value: worker # effect: NoSchedule +# One profile controls all compute observability defaults. +observability: + profile: compute + # ============================================================================= # Optional Add-on Operators # ============================================================================= diff --git a/deploy/stacks/nvcf-compute-plane/helmfile.d/02-nvca.yaml.gotmpl b/deploy/stacks/nvcf-compute-plane/helmfile.d/02-nvca.yaml.gotmpl index 4f43d812e..cc0ed1323 100644 --- a/deploy/stacks/nvcf-compute-plane/helmfile.d/02-nvca.yaml.gotmpl +++ b/deploy/stacks/nvcf-compute-plane/helmfile.d/02-nvca.yaml.gotmpl @@ -23,7 +23,7 @@ environments: the raw list; output is the normalized list as YAML. */}} {{- define "nvca-operator.featureGateValues" -}} -{{- $featureGateValues := . | default list -}} +{{- $featureGateValues := dig "values" list . | default list -}} {{- $defaultFeatureGates := list "DynamicGPUDiscovery" "InfraResourceOverhead" @@ -32,6 +32,9 @@ environments: "EnforceHelmTaskResourceLimits" "EnforceContainerTaskResourceLimits" -}} +{{- if dig "byooEnabled" false . -}} +{{- $defaultFeatureGates = append $defaultFeatureGates "BYOObservability" -}} +{{- end -}} {{- range $gate := $defaultFeatureGates -}} {{- if not (or (has $gate $featureGateValues) (has (printf "-%s" $gate) $featureGateValues)) -}} {{- $featureGateValues = append $featureGateValues $gate -}} @@ -59,6 +62,11 @@ helmDefaults: {{- $registrationValues := readFile $registrationValuesPath | fromYaml | default dict }} {{- $registrationAgentMergeConfig := dig "agentConfig" "mergeConfig" "" $registrationValues }} {{- $environmentAgentMergeConfig := dig "agentConfig" "mergeConfig" "" .Values }} +{{- $observabilityProfile := dig "observability" "profile" "compute" .Values }} +{{- if not (has $observabilityProfile (list "disabled" "control" "compute" "all")) }} +{{- fail (printf "observability.profile must be disabled, control, compute, or all, got %q" $observabilityProfile) }} +{{- end }} +{{- $computeObservabilityEnabled := or (eq $observabilityProfile "compute") (eq $observabilityProfile "all") }} {{- $agentMergeConfig := dict }} {{- if $registrationAgentMergeConfig }} {{- $agentMergeConfig = mergeOverwrite $agentMergeConfig ($registrationAgentMergeConfig | fromYaml | default dict) }} @@ -104,6 +112,8 @@ releases: clusterName: {{ requiredEnv "CLUSTER_NAME" }} ncaId: {{ requiredEnv "NCA_ID" }} {{- $selfManaged := dig "nvcaOperator" "selfManaged" dict .Values.global }} + {{- $otelCollector := dig "otelCollector" dict $selfManaged }} + {{- $otelCollectorEnabled := dig "enabled" $computeObservabilityEnabled $otelCollector }} selfManaged: identitySource: psat {{- with dig "icmsServiceURL" "" $selfManaged }} @@ -125,7 +135,17 @@ releases: natsHostOverride: {{ . | quote }} {{- end }} featureGateValues: - {{- include "nvca-operator.featureGateValues" (dig "featureGateValues" list $selfManaged) | nindent 10 }} + {{- include "nvca-operator.featureGateValues" (dict + "values" (dig "featureGateValues" list $selfManaged) + "byooEnabled" $computeObservabilityEnabled) | nindent 10 }} + otelCollector: + enabled: {{ $otelCollectorEnabled }} + {{- with dig "imageRepository" "" $otelCollector }} + imageRepository: {{ . | quote }} + {{- end }} + {{- with dig "imageTag" "" $otelCollector }} + imageTag: {{ . | quote }} + {{- end }} imageCredHelper: imageRepository: {{ .Values.global.image.registry }}/{{ .Values.global.image.repository }}/nvcf-image-credential-helper {{- with dig "imageCredHelper" "imageTag" "" $selfManaged }} diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/self-managed-nvcfbackend-cm.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/self-managed-nvcfbackend-cm.yaml index 9c83c2ff5..ca5466e53 100644 --- a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/self-managed-nvcfbackend-cm.yaml +++ b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/self-managed-nvcfbackend-cm.yaml @@ -42,7 +42,7 @@ data: cloudProvider: "NCP" region: "us-west-1" attributes: [] - capabilities: ["DynamicGPUDiscovery","InfraResourceOverhead","EnforceHelmFunctionResourceLimits","EnforceContainerFunctionResourceLimits","EnforceHelmTaskResourceLimits","EnforceContainerTaskResourceLimits"] + capabilities: ["DynamicGPUDiscovery","InfraResourceOverhead","EnforceHelmFunctionResourceLimits","EnforceContainerFunctionResourceLimits","EnforceHelmTaskResourceLimits","EnforceContainerTaskResourceLimits","BYOObservability"] gpusB64: "" cacheCSIVolumeMountOptionsConfig: disabled: false @@ -67,7 +67,7 @@ data: imageRepository: "nvcr.io/0651155215864979/ncp-dev/samba" imageTag: "1.0.5" otelCollector: - enabled: false + enabled: true imageConfig: repository: "nvcr.io/nvidia/nvcf-byoc/nvcf-otel-collector" tag: "0.143.2" diff --git a/deploy/stacks/nvcf-compute-plane/tests/compare-golden.sh b/deploy/stacks/nvcf-compute-plane/tests/compare-golden.sh new file mode 100755 index 000000000..3605b1c69 --- /dev/null +++ b/deploy/stacks/nvcf-compute-plane/tests/compare-golden.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +set -euo pipefail + +expected_dir="${1:?expected golden directory is required}" +actual_dir="${2:?actual manifest directory is required}" +work_dir="$(mktemp -d)" +trap 'rm -rf "$work_dir"' EXIT + +index_tree() { + local root_dir="$1" + local output_file="$2" + + find "$root_dir" -type f -print | + while IFS= read -r file; do + relative_path="${file#"$root_dir"/}" + normalized_path="$( + printf '%s\n' "$relative_path" | + sed -E 's#^([^/]+)-[0-9a-f]{8}-#\1-HASH-#' + )" + printf '%s %s\n' "$(git hash-object "$file")" "$normalized_path" + done | + sort >"$output_file" +} + +index_tree "$expected_dir" "$work_dir/expected" +index_tree "$actual_dir" "$work_dir/actual" +diff -u "$work_dir/expected" "$work_dir/actual" diff --git a/deploy/stacks/nvcf-compute-plane/tests/observability-profile.sh b/deploy/stacks/nvcf-compute-plane/tests/observability-profile.sh new file mode 100755 index 000000000..63a6ad097 --- /dev/null +++ b/deploy/stacks/nvcf-compute-plane/tests/observability-profile.sh @@ -0,0 +1,105 @@ +#!/usr/bin/env bash +set -euo pipefail + +stack_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +work_dir="$(mktemp -d)" +cluster_name="observability-profile-test-$$" +registration_values="$stack_dir/out/$cluster_name-register-values.yaml" +trap 'rm -rf "$work_dir"; rm -f "$registration_values"' EXIT + +fail() { + echo "observability-profile: $*" >&2 + exit 1 +} + +mkdir -p "$stack_dir/out" +cp "$stack_dir/testdata/registration/ncp-local-register-values.yaml" \ + "$registration_values" + +render_values() { + local profile="$1" + local output_file="$2" + local profile_args=() + shift 2 + + if [[ "$profile" != "default" ]]; then + profile_args=(--state-values-set "observability.profile=$profile") + fi + + HELMFILE_ENV=base \ + CLUSTER_NAME="$cluster_name" \ + NCA_ID=nvcf-default \ + helmfile \ + --file "$stack_dir/helmfile.d/02-nvca.yaml.gotmpl" \ + --environment default \ + "${profile_args[@]}" \ + "$@" \ + --selector name=nvca-operator \ + write-values \ + --output-file-template "$output_file" >/dev/null +} + +collector_enabled() { + awk ' + /^selfManaged:$/ { self_managed = 1; next } + self_managed && /^ otelCollector:$/ { collector = 1; next } + collector && /^ enabled:/ { print $2; exit } + ' "$1" +} + +has_byoo_gate() { + grep -q '^[[:space:]]*- BYOObservability$' "$1" +} + +for profile in default disabled control compute all; do + values="$work_dir/$profile.yaml" + render_values "$profile" "$values" + + case "$profile" in + default|compute|all) + test "$(collector_enabled "$values")" = "true" || + fail "$profile profile did not enable the NVCA collector" + has_byoo_gate "$values" || + fail "$profile profile did not enable BYOObservability" + ;; + disabled|control) + test "$(collector_enabled "$values")" = "false" || + fail "$profile profile enabled the NVCA collector" + if has_byoo_gate "$values"; then + fail "$profile profile enabled BYOObservability" + fi + ;; + esac +done + +render_values compute "$work_dir/compute-overrides.yaml" \ + --state-values-set global.nvcaOperator.selfManaged.otelCollector.enabled=false \ + --state-values-set-string 'global.nvcaOperator.selfManaged.featureGateValues[0]=-BYOObservability' + +test "$(collector_enabled "$work_dir/compute-overrides.yaml")" = "false" || + fail "explicit collector disable did not override the compute profile" +if has_byoo_gate "$work_dir/compute-overrides.yaml"; then + fail "explicit BYOObservability disable did not override the compute profile" +fi +grep -q '^[[:space:]]*- -BYOObservability$' \ + "$work_dir/compute-overrides.yaml" || + fail "explicit BYOObservability disable was not preserved" + +render_values control "$work_dir/control-overrides.yaml" \ + --state-values-set global.nvcaOperator.selfManaged.otelCollector.enabled=true \ + --state-values-set-string 'global.nvcaOperator.selfManaged.featureGateValues[0]=BYOObservability' + +test "$(collector_enabled "$work_dir/control-overrides.yaml")" = "true" || + fail "explicit collector enable did not override the control profile" +has_byoo_gate "$work_dir/control-overrides.yaml" || + fail "explicit BYOObservability enable did not override the control profile" + +if render_values invalid "$work_dir/invalid.yaml" \ + >"$work_dir/invalid.log" 2>&1; then + fail "invalid observability profile was accepted" +fi +grep -q 'observability.profile must be disabled, control, compute, or all' \ + "$work_dir/invalid.log" || + fail "invalid profile did not return the expected error" + +echo "observability-profile: all checks passed" diff --git a/deploy/stacks/observability/AGENTS.md b/deploy/stacks/observability/AGENTS.md index d71853a8c..62093aae1 100644 --- a/deploy/stacks/observability/AGENTS.md +++ b/deploy/stacks/observability/AGENTS.md @@ -19,8 +19,8 @@ deployments. `observability.profile`; do not expose parallel plane booleans. - Enabled profiles install shared components once by default. Fine-grained component modes are `install`, `existing`, and `disabled`. -- Derive NVCA BYOO support from compute and all profiles. Keep the NVCA - collector disabled by default. +- Derive the NVCA collector and `BYOObservability` feature gate from `compute` + and `all`. Preserve explicit compute-plane overrides. ## Key Files diff --git a/deploy/stacks/observability/Makefile b/deploy/stacks/observability/Makefile index d910aa875..5c3e6887f 100644 --- a/deploy/stacks/observability/Makefile +++ b/deploy/stacks/observability/Makefile @@ -30,6 +30,7 @@ test: --set functionautoscaler.image.repository=test/nvcf-function-autoscaler @tests/profile-defaults.sh @../self-managed/tests/observability-autoscaler.sh + @../nvcf-compute-plane/tests/observability-profile.sh install: @echo ">>> Installing observability stack" diff --git a/deploy/stacks/observability/README.md b/deploy/stacks/observability/README.md index b4fa54501..8167cfef2 100644 --- a/deploy/stacks/observability/README.md +++ b/deploy/stacks/observability/README.md @@ -25,6 +25,10 @@ self-managed stack defaults to `control`. Profiles derive the plane behavior internally. There are no `planes.control.enabled` or `planes.compute.enabled` values. +The compute-plane consumer maps `compute` and `all` to +`selfManaged.otelCollector.enabled: true` and the `BYOObservability` NVCA +feature gate. Explicit collector and feature-gate values still win. + ## Overrides Profiles are defaults, not restrictions. Monitor groups and individual targets From a38018df9e28d0f0217733bb4f2f6347e1b9f6fa Mon Sep 17 00:00:00 2001 From: Kristina Pathak Date: Tue, 28 Jul 2026 14:45:57 -0700 Subject: [PATCH 11/13] docs(observability): clarify backend validation --- deploy/stacks/observability/README.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/deploy/stacks/observability/README.md b/deploy/stacks/observability/README.md index 8167cfef2..be86c5af2 100644 --- a/deploy/stacks/observability/README.md +++ b/deploy/stacks/observability/README.md @@ -98,13 +98,17 @@ profile cannot install individual components. ## Metrics backend and autoscaler -The bundled VictoriaMetrics endpoints are: +With the default `monitoring` namespace, the bundled VictoriaMetrics endpoints +are: ```text remote write: http://vmsingle.monitoring.svc.cluster.local:8428/api/v1/write PromQL: http://vmsingle.monitoring.svc.cluster.local:8428 ``` +If the VictoriaMetrics namespace is overridden, replace `monitoring` in these +hostnames with the configured namespace. + An external backend always requires `remoteWriteEndpoint`; `control` and `all` also require `promqlEndpoint`. Authentication modes are `none`, `token`, and `mtls`. Token mode requires `authnEndpoint`; mTLS requires @@ -135,4 +139,5 @@ also requires NetworkPolicy reachability to the collector. ```sh make template HELMFILE_ENV=local make test +git diff --check ``` From 744b8e083bc06095b6f37229dbff9650d5786d8f Mon Sep 17 00:00:00 2001 From: Kristina Pathak Date: Tue, 28 Jul 2026 14:55:57 -0700 Subject: [PATCH 12/13] test(observability): cover collector image overrides --- .../nvcf-compute-plane/tests/observability-profile.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/deploy/stacks/nvcf-compute-plane/tests/observability-profile.sh b/deploy/stacks/nvcf-compute-plane/tests/observability-profile.sh index 63a6ad097..49201ae8e 100755 --- a/deploy/stacks/nvcf-compute-plane/tests/observability-profile.sh +++ b/deploy/stacks/nvcf-compute-plane/tests/observability-profile.sh @@ -74,6 +74,8 @@ done render_values compute "$work_dir/compute-overrides.yaml" \ --state-values-set global.nvcaOperator.selfManaged.otelCollector.enabled=false \ + --state-values-set-string global.nvcaOperator.selfManaged.otelCollector.imageRepository=registry.example.com/nvcf/collector \ + --state-values-set-string global.nvcaOperator.selfManaged.otelCollector.imageTag=test-tag \ --state-values-set-string 'global.nvcaOperator.selfManaged.featureGateValues[0]=-BYOObservability' test "$(collector_enabled "$work_dir/compute-overrides.yaml")" = "false" || @@ -84,6 +86,12 @@ fi grep -q '^[[:space:]]*- -BYOObservability$' \ "$work_dir/compute-overrides.yaml" || fail "explicit BYOObservability disable was not preserved" +grep -Eq '^ imageRepository: "?registry\.example\.com/nvcf/collector"?$' \ + "$work_dir/compute-overrides.yaml" || + fail "explicit collector image repository was not preserved" +grep -Eq '^ imageTag: "?test-tag"?$' \ + "$work_dir/compute-overrides.yaml" || + fail "explicit collector image tag was not preserved" render_values control "$work_dir/control-overrides.yaml" \ --state-values-set global.nvcaOperator.selfManaged.otelCollector.enabled=true \ From d728e6390c31a1f292c58ac9557ce045000b4dfe Mon Sep 17 00:00:00 2001 From: Kristina Pathak Date: Tue, 28 Jul 2026 15:43:35 -0700 Subject: [PATCH 13/13] fix(observability): allow target allocator secret discovery --- .../charts/nvcf-otel-collector/templates/rbac.yaml | 1 + deploy/stacks/observability/tests/profile-defaults.sh | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/deploy/stacks/observability/charts/nvcf-otel-collector/templates/rbac.yaml b/deploy/stacks/observability/charts/nvcf-otel-collector/templates/rbac.yaml index 5ff1d4696..95754c1d6 100644 --- a/deploy/stacks/observability/charts/nvcf-otel-collector/templates/rbac.yaml +++ b/deploy/stacks/observability/charts/nvcf-otel-collector/templates/rbac.yaml @@ -69,6 +69,7 @@ rules: - pods - configmaps - namespaces + - secrets verbs: - get - list diff --git a/deploy/stacks/observability/tests/profile-defaults.sh b/deploy/stacks/observability/tests/profile-defaults.sh index a0ec2524c..020bb1f32 100755 --- a/deploy/stacks/observability/tests/profile-defaults.sh +++ b/deploy/stacks/observability/tests/profile-defaults.sh @@ -111,6 +111,8 @@ helm template default-monitors "$stack_dir/charts/nvcf-default-monitors" \ --set controlPlane.enabled=true >"$work_dir/chart-control-defaults.yaml" helm template default-monitors "$stack_dir/charts/nvcf-default-monitors" \ --set computePlane.enabled=true >"$work_dir/chart-compute-defaults.yaml" +helm template otel-collector "$stack_dir/charts/nvcf-otel-collector" \ + >"$work_dir/chart-collector-defaults.yaml" control_manifests="$(find "$work_dir/control" -type f -name '*.yaml' -print)" compute_manifests="$(find "$work_dir/compute" -type f -name '*.yaml' -print)" @@ -121,6 +123,10 @@ compute_worker_disabled_manifests="$(find "$work_dir/compute-worker-disabled" -t chart_control_manifests="$work_dir/chart-control-defaults.yaml" chart_compute_manifests="$work_dir/chart-compute-defaults.yaml" worker_monitor_manifest="$work_dir/chart-worker-podmonitor.yaml" +collector_manifests="$work_dir/chart-collector-defaults.yaml" + +grep -q '^ - secrets$' "$collector_manifests" || + fail "Target Allocator RBAC must allow referenced Secret discovery" sed -n '/name: nvcf-default-monitors-worker/,$p' \ "$chart_compute_manifests" >"$worker_monitor_manifest"