test(bdd): cover all observability profile

tests/bdd/features/observability-all.feature

@@ -0,0 +1,140 @@
+@observability @all @ncp-local @single-cluster @helmfile
+Feature: Install local Helmfile observability for both planes
+  As a self-managed NVCF operator,
+  I want to install the all observability profile on one local cluster,
+  so that the control and compute planes share one metrics stack and expose
+  both monitor families.
+
+  Background:
+    Given environment variable "NGC_API_KEY" is set
+    And environment variable "SAMPLE_NGC_ORG" is set
+    And environment variable "SAMPLE_NGC_TEAM" is set
+    And environment variable "NVCF_CLI" is set
+    And environment variable "REPO_ROOT" is set
+    # Helmfile pulls OCI charts during installation. Keep $NGC_API_KEY unbraced
+    # so the BDD runner does not expand it into command logs.
+    And command has succeeded:
+      """
+      bash -c 'set -eo pipefail; printf %s "$NGC_API_KEY" | helm registry login nvcr.io --username "\$oauthtoken" --password-stdin'
+      """
+    # Configure the control-plane stack and its shared observability child.
+    And I copy the file "tests/bdd/fixtures/self-managed-local-bdd.yaml" to "deploy/stacks/self-managed/environments/local-bdd-observability-all.yaml"
+    And I update yaml file "deploy/stacks/self-managed/environments/local-bdd-observability-all.yaml" with keys:
+      | global.imagePullSecrets[0].name | nvcr-pull-secret                     |
+      | global.helm.sources.repository  | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM} |
+      | global.image.repository         | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM} |
+      | observability.profile           | all                                  |
+      | functionAutoscaler.image.tag    | 1.18.10                              |
+    # Give the shared observability Helmfile the same named environment.
+    And I copy the file "tests/bdd/fixtures/self-managed-local-bdd.yaml" to "deploy/stacks/observability/environments/local-bdd-observability-all.yaml"
+    And I update yaml file "deploy/stacks/observability/environments/local-bdd-observability-all.yaml" with keys:
+      | global.imagePullSecrets[0].name | nvcr-pull-secret                     |
+      | global.helm.sources.repository  | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM} |
+      | global.image.repository         | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM} |
+      | observability.profile           | all                                  |
+    # Configure NVCA to join the same cluster and enable its collector.
+    And I copy the file "tests/bdd/fixtures/nvcf-compute-plane-local-bdd.yaml" to "deploy/stacks/nvcf-compute-plane/environments/local-bdd-observability-all.yaml"
+    And I update yaml file "deploy/stacks/nvcf-compute-plane/environments/local-bdd-observability-all.yaml" with keys:
+      | global.imagePullSecrets[0].name                               | nvcr-pull-secret                                                  |
+      | global.helm.sources.repository                                | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM}                              |
+      | global.image.repository                                       | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM}                              |
+      | global.nvcaOperator.selfManaged.otelCollector.imageRepository | nvcr.io/${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM}/nvcf-otel-collector |
+      | observability.profile                                         | all                                                               |
+    And I copy the file "deploy/stacks/self-managed/secrets/secrets.yaml.template" to "deploy/stacks/self-managed/secrets/local-bdd-observability-all-secrets.yaml"
+    And I substitute "REPLACE_WITH_BASE64_DOCKER_CREDENTIAL" in file "deploy/stacks/self-managed/secrets/local-bdd-observability-all-secrets.yaml" with base64 of "$oauthtoken:${NGC_API_KEY}"
+    # Conflict precheck: the split topology claims host ports used by the
+    # single-cluster topology. From the repository root, run
+    # `make -C tools/ncp-local-cluster destroy-all-ncp-local SHELL=/bin/bash`
+    # before retrying.
+    Given I run command "k3d cluster get ncp-local-cp"
+    And the command exit code should be 1
+    And a single-cluster ncp-local cluster is running
+    # Keep every install and registration operation off the ambient kube context.
+    And command has succeeded:
+      """
+      k3d kubeconfig merge ncp-local --output ${REPO_ROOT}/tests/bdd/out/ncp-local-observability-all-kubeconfig.yaml --overwrite --kubeconfig-switch-context=false
+      """
+    And the "nvcr-pull-secret" image pull secret exists in namespaces using context "k3d-ncp-local":
+      | cassandra-system |
+      | nats-system      |
+      | nvcf             |
+      | api-keys         |
+      | ess              |
+      | sis              |
+      | vault-system     |
+      | nvca-operator    |
+      | cert-manager     |
+      | monitoring       |
+
+  Scenario: All profile installs one shared stack with both monitor families
+    When I run command:
+      """
+      make -C deploy/stacks/self-managed install HELMFILE_ENV=local-bdd-observability-all KUBECONFIG_FILE=${REPO_ROOT}/tests/bdd/out/ncp-local-observability-all-kubeconfig.yaml
+      """
+    Then the command exit code should be 0
+
+    When I run command:
+      """
+      make -C deploy/stacks/nvcf-compute-plane register-cluster CLUSTER_NAME=ncp-local KUBECONFIG_FILE=${REPO_ROOT}/tests/bdd/out/ncp-local-observability-all-kubeconfig.yaml NVCF_CLI=${NVCF_CLI} NVCF_CLI_CONFIG=${REPO_ROOT}/tests/bdd/fixtures/nvcf-cli-local.yaml
+      """
+    Then the command exit code should be 0
+    And file "deploy/stacks/nvcf-compute-plane/registration/ncp-local-register-values.yaml" should exist
+    And yaml file "deploy/stacks/nvcf-compute-plane/registration/ncp-local-register-values.yaml" key "clusterID" should not be empty
+    And yaml file "deploy/stacks/nvcf-compute-plane/registration/ncp-local-register-values.yaml" key "clusterGroupID" should not be empty
+
+    When I run command:
+      """
+      make -C deploy/stacks/nvcf-compute-plane install CLUSTER_NAME=ncp-local HELMFILE_ENV=local-bdd-observability-all KUBECONFIG_FILE=${REPO_ROOT}/tests/bdd/out/ncp-local-observability-all-kubeconfig.yaml NVCF_CLI=${NVCF_CLI} NVCF_CLI_CONFIG=${REPO_ROOT}/tests/bdd/fixtures/nvcf-cli-local.yaml
+      """
+    Then the command exit code should be 0
+
+    # Self-hosted NVCA intentionally creates an empty NGC service-key secret.
+    # Supply the existing local credential so the NVCA collector can start,
+    # then restart NVCA to consume it. Keep $NGC_API_KEY out of command logs.
+    And command has succeeded:
+      """
+      bash -c 'set -eo pipefail; printf %s "$NGC_API_KEY" | kubectl --context k3d-ncp-local create secret generic ngc-service-api-key --namespace nvca-system --from-file=ngc-service-api-key=/dev/stdin --dry-run=client -o yaml | kubectl --context k3d-ncp-local apply -f -'
+      """
+    And command has succeeded:
+      """
+      kubectl --context k3d-ncp-local delete pod --namespace nvca-system --selector app.kubernetes.io/name=nvca --wait=false
+      """
+
+    # Revision 1 proves the compute install did not reinstall or upgrade the
+    # shared observability releases created by the control-plane install.
+    When I run command "helm list --all-namespaces --kube-context k3d-ncp-local -o json"
+    Then the json output should contain rows:
+      | name                     | namespace     | revision | status   |
+      | prometheus-operator-crds | monitoring    | 1        | deployed |
+      | opentelemetry-operator   | monitoring    | 1        | deployed |
+      | victoria-metrics         | monitoring    | 1        | deployed |
+      | otel-collector           | monitoring    | 1        | deployed |
+      | default-monitors         | monitoring    | 1        | deployed |
+      | nvca-operator            | nvca-operator | 1        | deployed |
+
+    When I run command "kubectl rollout status deployment/nvca-operator -n nvca-operator --context k3d-ncp-local --timeout=10m"
+    Then the command exit code should be 0
+    When I run command "kubectl wait nvcfbackend ncp-local -n nvca-operator --context k3d-ncp-local --for=jsonpath={.status.agentStatus}=healthy --timeout=10m"
+    Then the command exit code should be 0
+
+    When I run command "kubectl get opentelemetrycollector nvcf-observability -n monitoring --context k3d-ncp-local -o jsonpath='{.spec.targetAllocator.enabled}'"
+    Then the command exit code should be 0
+    And the command output should contain "true"
+
+    Then these ServiceMonitors should exist in namespace "monitoring" using context "k3d-ncp-local":
+      | name                                             |
+      | nvcf-default-monitors-state-metrics              |
+      | nvcf-default-monitors-grpc-proxy                  |
+      | nvcf-default-monitors-llm-api-gateway             |
+      | nvcf-default-monitors-invocation-service          |
+      | nvcf-default-monitors-nvca                        |
+
+    When I run command "kubectl get podmonitor/nvcf-default-monitors-dcgm podmonitor/nvcf-default-monitors-worker --namespace monitoring --context k3d-ncp-local"
+    Then the command exit code should be 0
+
+    When I run command:
+      """
+      bash -c 'set -eo pipefail; helm get values nvca-operator --namespace nvca-operator --kube-context k3d-ncp-local -o json | jq -r ".selfManaged.otelCollector.enabled"'
+      """
+    Then the command exit code should be 0
+    And the command output should contain "true"

tests/bdd/fixtures/self-managed-local-bdd.yaml

@@ -78,8 +78,7 @@ addons:
             "request_algorithms": {
               "power-of-two": "power-of-two",
               "round-robin": "round-robin",
-              "random": "random",
-              "wait-and-widen": "wait-and-widen"
+              "random": "random"
             }
           }
 

tests/bdd/godog_test.go

@@ -680,6 +680,147 @@ func observabilityComputeHelmListJSON() string {
 ]`
 }
 
+// TestObservabilityAllFeatureFileWiresToSteps runs the all-profile feature
+// against a fake runner. It keeps every cluster operation on the explicit
+// local context and verifies that one shared stack serves both monitor sets.
+func TestObservabilityAllFeatureFileWiresToSteps(t *testing.T) {
+	const (
+		registryLoginCommand   = `bash -c 'set -eo pipefail; printf %s "$NGC_API_KEY" | helm registry login nvcr.io --username "\$oauthtoken" --password-stdin'`
+		serviceMonitorsCommand = "kubectl get servicemonitor/nvcf-default-monitors-state-metrics" +
+			" servicemonitor/nvcf-default-monitors-grpc-proxy" +
+			" servicemonitor/nvcf-default-monitors-llm-api-gateway" +
+			" servicemonitor/nvcf-default-monitors-invocation-service" +
+			" servicemonitor/nvcf-default-monitors-nvca" +
+			" --namespace monitoring --context k3d-ncp-local"
+		podMonitorsCommand = "kubectl get podmonitor/nvcf-default-monitors-dcgm" +
+			" podmonitor/nvcf-default-monitors-worker" +
+			" --namespace monitoring --context k3d-ncp-local"
+		collectorEnabledCommand = `bash -c 'set -eo pipefail; helm get values nvca-operator --namespace nvca-operator --kube-context k3d-ncp-local -o json | jq -r ".selfManaged.otelCollector.enabled"'`
+		serviceKeyCommand       = `bash -c 'set -eo pipefail; printf %s "$NGC_API_KEY" |` +
+			` kubectl --context k3d-ncp-local create secret generic ngc-service-api-key` +
+			` --namespace nvca-system --from-file=ngc-service-api-key=/dev/stdin --dry-run=client -o yaml |` +
+			` kubectl --context k3d-ncp-local apply -f -'`
+		restartNVCACommand = "kubectl --context k3d-ncp-local delete pod --namespace nvca-system --selector app.kubernetes.io/name=nvca --wait=false"
+	)
+	t.Setenv("NGC_API_KEY", "test-key")
+	t.Setenv("SAMPLE_NGC_ORG", "test-org")
+	t.Setenv("SAMPLE_NGC_TEAM", "test-team")
+	t.Setenv("NVCF_CLI", "/usr/bin/nvcf-cli")
+	t.Setenv("REPO_ROOT", "/repo-root-placeholder")
+	suite := newWiringSuite(t, newFakeRunner(map[string]harness.Result{
+		registryLoginCommand:           {ExitCode: 0},
+		"k3d cluster get ncp-local-cp": {ExitCode: 1},
+		serviceMonitorsCommand:         {ExitCode: 0},
+		podMonitorsCommand:             {ExitCode: 0},
+		collectorEnabledCommand:        {ExitCode: 0, Stdout: "true\n"},
+		serviceKeyCommand:              {ExitCode: 0},
+		restartNVCACommand:             {ExitCode: 0},
+		"helm list --all-namespaces --kube-context k3d-ncp-local -o json": {
+			ExitCode: 0,
+			Stdout:   observabilityAllHelmListJSON(),
+		},
+		"kubectl get opentelemetrycollector nvcf-observability -n monitoring --context k3d-ncp-local -o jsonpath='{.spec.targetAllocator.enabled}'": {
+			ExitCode: 0,
+			Stdout:   "true",
+		},
+	}))
+	seedHelmfileLocalBDDFixture(t, suite.Config.RepoRoot)
+	seedComputePlaneLocalBDDFixture(t, suite.Config.RepoRoot)
+	seedStackSecretsTemplate(t, suite.Config.RepoRoot)
+	writeHelmfileRegisterValues(t, suite.Config.RepoRoot)
+
+	sc := steps.NewScenarioContext(suite)
+	featurePath := mustResolveFeaturePath(t, "observability-all.feature")
+	var out strings.Builder
+	status := godog.TestSuite{
+		Name: "observability-all-wiring",
+		ScenarioInitializer: func(ctx *godog.ScenarioContext) {
+			steps.RegisterAll(ctx, sc)
+		},
+		Options: &godog.Options{
+			Format: "pretty",
+			Paths:  []string{featurePath},
+			Strict: true,
+			Output: &out,
+		},
+	}.Run()
+	if status != 0 {
+		t.Fatalf("godog suite status = %d\n%s", status, out.String())
+	}
+
+	runs := suite.Runner.(*fakeRunner).runs
+	for _, command := range []string{
+		registryLoginCommand,
+		serviceMonitorsCommand,
+		podMonitorsCommand,
+		collectorEnabledCommand,
+		serviceKeyCommand,
+		restartNVCACommand,
+	} {
+		if !commandRanExactly(runs, command) {
+			t.Fatalf("exact command was never invoked: %s", command)
+		}
+	}
+	for _, commandFragment := range []string{
+		"deploy/stacks/self-managed install HELMFILE_ENV=local-bdd-observability-all KUBECONFIG_FILE=/repo-root-placeholder/tests/bdd/out/ncp-local-observability-all-kubeconfig.yaml",
+		"register-cluster CLUSTER_NAME=ncp-local KUBECONFIG_FILE=/repo-root-placeholder/tests/bdd/out/ncp-local-observability-all-kubeconfig.yaml",
+		"deploy/stacks/nvcf-compute-plane install CLUSTER_NAME=ncp-local HELMFILE_ENV=local-bdd-observability-all KUBECONFIG_FILE=/repo-root-placeholder/tests/bdd/out/ncp-local-observability-all-kubeconfig.yaml",
+	} {
+		if !commandRanThatContains(runs, commandFragment) {
+			t.Fatalf("command containing %q was never invoked", commandFragment)
+		}
+	}
+	for _, run := range runs {
+		if strings.HasPrefix(run, "kubectl apply -f ") {
+			t.Fatalf("manifest apply relied on the ambient kube context: %s", run)
+		}
+		if strings.Contains(run, "test-key") {
+			t.Fatalf("NGC API key leaked into command arguments: %s", run)
+		}
+	}
+
+	for _, stack := range []string{"self-managed", "observability", "nvcf-compute-plane"} {
+		environmentPath := filepath.Join(suite.Config.RepoRoot, "deploy", "stacks", stack, "environments", "local-bdd-observability-all.yaml")
+		profile, found, err := dsl.ReadYAMLKey(environmentPath, "observability.profile")
+		if err != nil {
+			t.Fatalf("read %s observability profile: %v", stack, err)
+		}
+		if !found || profile != "all" {
+			t.Fatalf("%s observability profile = %q, found = %t; want all", stack, profile, found)
+		}
+	}
+
+	assertions := []struct {
+		stack string
+		key   string
+		want  string
+	}{
+		{stack: "self-managed", key: "functionAutoscaler.image.tag", want: "1.18.10"},
+		{stack: "nvcf-compute-plane", key: "global.nvcaOperator.selfManaged.otelCollector.imageRepository", want: "nvcr.io/test-org/test-team/nvcf-otel-collector"},
+	}
+	for _, assertion := range assertions {
+		environmentPath := filepath.Join(suite.Config.RepoRoot, "deploy", "stacks", assertion.stack, "environments", "local-bdd-observability-all.yaml")
+		got, found, err := dsl.ReadYAMLKey(environmentPath, assertion.key)
+		if err != nil {
+			t.Fatalf("read %s override %s: %v", assertion.stack, assertion.key, err)
+		}
+		if !found || got != assertion.want {
+			t.Fatalf("%s override %s = %q, found = %t; want %q", assertion.stack, assertion.key, got, found, assertion.want)
+		}
+	}
+}
+
+func observabilityAllHelmListJSON() string {
+	return `[
+{"name":"prometheus-operator-crds","namespace":"monitoring","revision":"1","status":"deployed"},
+{"name":"opentelemetry-operator","namespace":"monitoring","revision":"1","status":"deployed"},
+{"name":"victoria-metrics","namespace":"monitoring","revision":"1","status":"deployed"},
+{"name":"otel-collector","namespace":"monitoring","revision":"1","status":"deployed"},
+{"name":"default-monitors","namespace":"monitoring","revision":"1","status":"deployed"},
+{"name":"nvca-operator","namespace":"nvca-operator","revision":"1","status":"deployed"}
+]`
+}
+
 // TestMultiClusterHelmfileFeatureFileWiresToSteps runs
 // multi-cluster-helmfile.feature against a fake runner. The same
 // fixture seeds and canned helm-list outputs cover the scenarios;
@@ -1407,6 +1548,15 @@ func TestObservabilityCompute(t *testing.T) {
 	runLiveFeature(t, "observability-compute.feature")
 }
 
+// TestObservabilityAll is the live entry point for both observability planes
+// on the local single-cluster topology. Skipped under -short.
+func TestObservabilityAll(t *testing.T) {
+	if testing.Short() {
+		t.Skip("live run skipped under -short")
+	}
+	runLiveFeature(t, "observability-all.feature")
+}
+
 // TestSingleClusterHelmfileUpstreamImages is the live entry point for the
 // focused Docker Hub supporting-image override feature. Skipped under -short.
 func TestSingleClusterHelmfileUpstreamImages(t *testing.T) {